How to remove border around BottomNavigationView

You can solve this problem in two steps:

Step 1: Create a drawable in the shape of a rectangle with a transparent color

res\drawable\transparent_rect.xml

<?xml version="1.0" encoding="utf-8"?>
<shape xmlns:android="http://schemas.android.com/apk/res/android"
    android:shape="rectangle" >
    <solid android:color="#00000000" />
</shape>

Step 2: Set the background of BottomNavigationViewthis drawable usingandroid:background

android:background="@drawable/transparent_rect"

Apply this to your layout:

<?xml version="1.0" encoding="utf-8"?>
<androidx.coordinatorlayout.widget.CoordinatorLayout
xmlns:android="http://schemas.android.com/apk/res/android"
android:layout_width="match_parent"
android:layout_height="match_parent"
xmlns:tools="http://schemas.android.com/tools"
xmlns:app="http://schemas.android.com/apk/res-auto"
tools:context=".main.BottomNavigation">

    <FrameLayout
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        android:id="@+id/frameContainer"
        app:layout_constraintBottom_toTopOf = "@id/bottomAppBar">
    </FrameLayout>

    <com.google.android.material.bottomappbar.BottomAppBar
        android:id="@+id/bottomAppBar"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        app:fabCradleMargin="10dp"
        app:fabCradleVerticalOffset="10dp"
        app:fabCradleRoundedCornerRadius="20dp"
        android:layout_gravity="bottom">     
            
        <com.google.android.material.bottomnavigation.BottomNavigationView
            android:id="@+id/bottomNavigationView"
            android:layout_width="match_parent"
            android:layout_height="match_parent"
            app:menu="@menu/bottom_nav_menu"
            android:layout_marginEnd="16dp"
            app:popupTheme="@style/MyDarkToolbarStyle"
            android:background="@drawable/transparent_rect"
            app:backgroundTint="@android:color/transparent"/>
    </com.google.android.material.bottomappbar.BottomAppBar>         

    <com.google.android.material.floatingactionbutton.FloatingActionButton
        android:id="@+id/fab"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:backgroundTint="@color/white"
        android:src="@drawable/ic_explore"
        app:layout_anchor="@id/bottomAppBar" />
    </androidx.coordinatorlayout.widget.CoordinatorLayout>

Preview: