How can I make the correct image go into the corresponding imageview?

Here I want to drag and drop the correct image (ie letter A) from the left to the image on the right and increase the score by 1. But after writing the following code, all images disappear when they fall out of view. And all images make the score go up with the correct score. Can anyone help me?

My Gameactivity code:

    img1 = (ImageView) findViewById(R.id.ic_one);
    img2 = (ImageView) findViewById(R.id.ic_two);
    img3 = (ImageView) findViewById(R.id.ic_three);
    img4 = (ImageView) findViewById(R.id.ic_four);
    imgmain = (ImageView) findViewById(R.id.ic_main);

    img1.setImageResource(R.mipmap.a1);
img2.setImageResource(R.mipmap.a2);
img3.setImageResource(R.mipmap.a3);
img4.setImageResource(R.mipmap.a4);
imgmain.setImageResource(R.mipmap.a1);

    //setting touch listener

    img1.setOnTouchListener(listenTouch);
    img2.setOnTouchListener(listenTouch);
    img3.setOnTouchListener(listenTouch);
    img4.setOnTouchListener(listenTouch);
    imgmain.setOnDragListener(listenDrag);

    display = (TextView) findViewById(R.id.txtScore);

}

public View.OnTouchListener listenTouch = new View.OnTouchListener() {
    @Override
    public boolean onTouch(View v, MotionEvent event) {
        if (event.getAction() == MotionEvent.ACTION_DOWN) {
            //setup drag
            ClipData data = ClipData.newPlainText("", "");
            View.DragShadowBuilder shadowBuilder = new View.DragShadowBuilder(v);
            v.startDrag(data, shadowBuilder, v, 0);
           v.setVisibility(View.INVISIBLE);

            return true;
        } else {
            return false;
        }
    }
};
public View.OnDragListener listenDrag = new View.OnDragListener() {
    @Override
    public boolean onDrag(View v, DragEvent event) {

        switch (event.getAction()) {
            case DragEvent.ACTION_DRAG_STARTED:


            case DragEvent.ACTION_DRAG_ENTERED:
                //no action necessary
                break;
            case DragEvent.ACTION_DRAG_EXITED:
                //no action necessary

                break;
            case DragEvent.ACTION_DROP:
                //handle the dragged view being dropped over a drop view
               if(v.getId() == R.id.ic_main) {
                  View view = (View) event.getLocalState();
                  ViewGroup viewgroup = (ViewGroup) view.getParent();
                 viewgroup.removeView(view);

                  //change the text
                  score++;
                  display.setText("Your score is : "+score);


                  view.setVisibility(View.VISIBLE);
              } else {
                  View view = (View) event.getLocalState();
                  view.setVisibility(View.VISIBLE);
                  Context context = getApplicationContext();
                  Toast.makeText(context, "You can't drop the image here",
                          Toast.LENGTH_LONG).show();
                  break;
               }
                break;
            case DragEvent.ACTION_DRAG_ENDED:
                //no action necessary
                break;
            default:
                break;
        }

        return true;
    }
};

}

And my layout code:

<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools" android:layout_width="match_parent"
android:layout_height="match_parent" android:paddingLeft="@dimen/activity_horizontal_margin"
android:paddingRight="@dimen/activity_horizontal_margin"
android:paddingTop="@dimen/activity_vertical_margin"
android:paddingBottom="@dimen/activity_vertical_margin"
tools:context="com.example.bhuwan.firstproject.GameActivity"
android:id="@+id/layout_game">

<ImageView
    android:layout_width="150dp"
    android:layout_height="150dp"
    android:id="@+id/ic_main"
    android:layout_alignBottom="@+id/ic_two"
    android:layout_alignParentRight="true"
    android:layout_alignParentEnd="true"
    android:layout_marginBottom="36dp" />

<ImageView
    android:layout_width="100dp"
    android:layout_height="100dp"
    android:id="@+id/ic_one"
    android:layout_alignParentTop="true" />

<ImageView
    android:layout_width="100dp"
    android:layout_height="100dp"
    android:id="@+id/ic_two"
    android:layout_marginTop="143dp"
    android:layout_alignParentTop="true"
    android:layout_alignParentLeft="true"
    android:layout_alignParentStart="true" />

<ImageView
    android:layout_width="100dp"
    android:layout_height="100dp"
    android:id="@+id/ic_three"
    android:layout_alignParentTop="true"
    android:layout_toRightOf="@+id/ic_one"
    android:layout_toEndOf="@+id/ic_one" />

<ImageView
    android:layout_width="100dp"
    android:layout_height="100dp"
    android:id="@+id/ic_four"
    android:layout_alignTop="@+id/ic_two"
    android:layout_toRightOf="@+id/ic_two"
    android:layout_toEndOf="@+id/ic_two" />

<TextView
    android:layout_width="wrap_content"
    android:layout_height="wrap_content"
    android:textAppearance="?android:attr/textAppearanceLarge"
    android:text="@string/empty"
    android:id="@+id/txtScore"
    android:layout_marginBottom="60dp"
    android:layout_alignParentBottom="true"
    android:layout_alignParentLeft="true"
    android:layout_alignParentStart="true"
    android:layout_marginLeft="42dp"
    android:layout_marginStart="42dp" />