← back to Handbag Authentication
handbag_data/github_datasets/Pursearch/tutorials/Remove_Repetative_Data.ipynb
321 lines
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2020-01-22T04:25:44.407799Z",
"start_time": "2020-01-22T04:25:39.589243Z"
}
},
"outputs": [],
"source": [
"import os\n",
"import pathlib\n",
"import numpy as np\n",
"import os\n",
"import six.moves.urllib as urllib\n",
"import sys\n",
"import tarfile\n",
"import tensorflow as tf\n",
"import zipfile\n",
"\n",
"from collections import defaultdict\n",
"from io import StringIO\n",
"from matplotlib import pyplot as plt\n",
"from PIL import Image\n",
"from IPython.display import display"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Some images are repetative, to avoid repetative images appear in both training and testing, we check the images"
]
},
{
"cell_type": "code",
"execution_count": 62,
"metadata": {
"ExecuteTime": {
"end_time": "2020-01-22T06:08:32.340459Z",
"start_time": "2020-01-22T05:59:01.433316Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/jianinglu1/anaconda3/envs/datainsight/lib/python3.7/site-packages/ipykernel_launcher.py:10: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.\n",
" # Remove the CWD from sys.path while we load stuff.\n"
]
}
],
"source": [
"datadir = \"/Users/jianinglu1/Documents/GitHub/DataInsight/data/preprocessed/Coach/F57842\"\n",
"repet = []\n",
"for i in range(322):\n",
" if i not in repet and str(i) + \".jpg\" in os.listdir(datadir):\n",
" image1= Image.open(os.path.join(datadir, str(i) + \".jpg\"))\n",
" for j in range(i+1,322):\n",
" if j not in repet and str(j) + \".jpg\" in os.listdir(datadir):\n",
" image2= Image.open(os.path.join(datadir, str(j) + \".jpg\"))\n",
" try:\n",
" if (np.array(image1) == np.array(image2)).all():\n",
" repet.append(j)\n",
" except:\n",
" continue"
]
},
{
"cell_type": "code",
"execution_count": 69,
"metadata": {
"ExecuteTime": {
"end_time": "2020-01-22T16:54:58.957580Z",
"start_time": "2020-01-22T16:36:51.845452Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"192\n",
"240\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/jianinglu1/anaconda3/envs/datainsight/lib/python3.7/site-packages/ipykernel_launcher.py:14: DeprecationWarning: elementwise comparison failed; this will raise an error in the future.\n",
" \n"
]
}
],
"source": [
"datadir = \"/Users/jianinglu1/Documents/GitHub/DataInsight/data/preprocessed/Chanel/2.55_classicdoubleflapbag\"\n",
"list1 = [i for i in os.listdir(datadir) if i.startswith(\"2.55\")]\n",
"print(len(list1))\n",
"list2 = [i for i in os.listdir(datadir) if i.startswith(\"classicdoubleflapbag\")]\n",
"print(len(list2))\n",
"repet = []\n",
"for i in list1:\n",
" if i not in repet:\n",
" image1= Image.open(os.path.join(datadir, i))\n",
" for j in list2:\n",
" if j not in repet:\n",
" image2= Image.open(os.path.join(datadir, j))\n",
" try:\n",
" if (np.array(image1) == np.array(image2)).all():\n",
" repet.append(j)\n",
" except:\n",
" continue"
]
},
{
"cell_type": "code",
"execution_count": 70,
"metadata": {
"ExecuteTime": {
"end_time": "2020-01-22T17:08:21.643985Z",
"start_time": "2020-01-22T17:08:21.619300Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"18"
]
},
"execution_count": 70,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(repet)"
]
},
{
"cell_type": "code",
"execution_count": 73,
"metadata": {
"ExecuteTime": {
"end_time": "2020-01-22T17:10:13.295814Z",
"start_time": "2020-01-22T17:10:13.285941Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"414"
]
},
"execution_count": 73,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"192+ 240 -18"
]
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {
"ExecuteTime": {
"end_time": "2020-01-22T17:08:34.266040Z",
"start_time": "2020-01-22T17:08:34.229813Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"['classicdoubleflapbag_86.jpg',\n",
" 'classicdoubleflapbag_294.jpg',\n",
" 'classicdoubleflapbag_257.jpg',\n",
" 'classicdoubleflapbag_20.jpg',\n",
" 'classicdoubleflapbag_146.jpg',\n",
" 'classicdoubleflapbag_165.jpg',\n",
" 'classicdoubleflapbag_145.jpg',\n",
" 'classicdoubleflapbag_220.jpg',\n",
" 'classicdoubleflapbag_5.jpg',\n",
" 'classicdoubleflapbag_174.jpg',\n",
" 'classicdoubleflapbag_30.jpg',\n",
" 'classicdoubleflapbag_307.jpg',\n",
" 'classicdoubleflapbag_199.jpg',\n",
" 'classicdoubleflapbag_407.jpg',\n",
" 'classicdoubleflapbag_188.jpg',\n",
" 'classicdoubleflapbag_421.jpg',\n",
" 'classicdoubleflapbag_442.jpg',\n",
" 'classicdoubleflapbag_406.jpg']"
]
},
"execution_count": 71,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"repet"
]
},
{
"cell_type": "code",
"execution_count": 72,
"metadata": {
"ExecuteTime": {
"end_time": "2020-01-22T17:09:13.978829Z",
"start_time": "2020-01-22T17:09:13.860213Z"
}
},
"outputs": [],
"source": [
"for i in repet:\n",
" if str(i) in os.listdir(datadir):\n",
" os.system(\"rm \" + os.path.join(datadir,str(i)))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"I need to merge the 2.55 and classic flap and I have a new dir is 2.55_classicdoubleflapbag"
]
},
{
"cell_type": "code",
"execution_count": 66,
"metadata": {
"ExecuteTime": {
"end_time": "2020-01-22T16:28:28.203945Z",
"start_time": "2020-01-22T16:28:24.603568Z"
}
},
"outputs": [],
"source": [
"olddir = os.getcwd()\n",
"datadir = \"/Users/jianinglu1/Documents/GitHub/DataInsight/data/preprocessed/Chanel/\"\n",
"os.chdir(datadir)\n",
"for i in os.listdir(datadir + \"2.55\"):\n",
" os.system(\"cp 2.55/\" + i + \" 2.55_classicdoubleflapbag/2.55_\" + i )\n",
"for i in os.listdir(datadir + \"classicdoubleflapbag\"):\n",
" os.system(\"cp classicdoubleflapbag/\" + i + \" 2.55_classicdoubleflapbag/classicdoubleflapbag_\" + i )\n",
"os.chdir(olddir)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": false
},
"varInspector": {
"cols": {
"lenName": 16,
"lenType": 16,
"lenVar": 40
},
"kernels_config": {
"python": {
"delete_cmd_postfix": "",
"delete_cmd_prefix": "del ",
"library": "var_list.py",
"varRefreshCmd": "print(var_dic_list())"
},
"r": {
"delete_cmd_postfix": ") ",
"delete_cmd_prefix": "rm(",
"library": "var_list.r",
"varRefreshCmd": "cat(var_dic_list()) "
}
},
"types_to_exclude": [
"module",
"function",
"builtin_function_or_method",
"instance",
"_Feature"
],
"window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}