{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "initial_id", "metadata": { "collapsed": true, "ExecuteTime": { "end_time": "2024-01-24T07:55:37.559571600Z", "start_time": "2024-01-24T07:55:35.286921700Z" } }, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 2, "outputs": [], "source": [ "import os" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T07:55:43.027207900Z", "start_time": "2024-01-24T07:55:43.011508100Z" } }, "id": "edeebedebcfbfaea" }, { "cell_type": "code", "execution_count": 3, "outputs": [], "source": [ "raw_dataset_path = \"./raw_dataset\"\n", "dataset_path = \"./dataset\"\n", "if not os.path.exists(dataset_path):\n", " os.makedirs(dataset_path)" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T07:55:43.994656600Z", "start_time": "2024-01-24T07:55:43.980635100Z" } }, "id": "97fcd29fcef2f6e2" }, { "cell_type": "code", "execution_count": 4, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['Channel_02.xlsx', 'Channel_03.xlsx', 'Channel_04.xlsx', 'Channel_05.xlsx', 'Channel_06.xlsx', 'Channel_07.xlsx', 'Channel_08.xlsx', 'Channel_09.xlsx', 'Channel_10.xlsx', 'Channel_11.xlsx']\n" ] } ], "source": [ "file_list = []\n", "if os.path.exists(raw_dataset_path):\n", " file_list = os.listdir(raw_dataset_path)\n", " if len(file_list) > 0:\n", " print(file_list)" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T07:55:47.321452800Z", "start_time": "2024-01-24T07:55:47.281687200Z" } }, "id": "34ef1d4940b070ac" }, { "cell_type": "code", "execution_count": 5, "outputs": [], "source": [ "user_dict = {}\n", "freq_point = {}\n", "file_temp = file_list[0]\n", "# month_2 = pd.read_excel()" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:05:28.328512700Z", "start_time": "2024-01-24T08:05:28.304339300Z" } }, "id": "8b430d34ef733a81" }, { "cell_type": "code", "execution_count": 6, "outputs": [ { "data": { "text/plain": "'Channel_02.xlsx'" }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "file_temp" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:05:30.846556100Z", "start_time": "2024-01-24T08:05:30.770308100Z" } }, "id": "df5d8247ba887864" }, { "cell_type": "code", "execution_count": 7, "outputs": [ { "data": { "text/plain": "" }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "os.path" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:06:07.946601Z", "start_time": "2024-01-24T08:06:07.923697900Z" } }, "id": "5fbd685fb403d624" }, { "cell_type": "code", "execution_count": 8, "outputs": [ { "data": { "text/plain": "'./raw_dataset\\\\Channel_02.xlsx'" }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "os.path.join(raw_dataset_path, file_temp)" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:06:37.377941200Z", "start_time": "2024-01-24T08:06:37.355576600Z" } }, "id": "17b36e1bf46376b7" }, { "cell_type": "code", "execution_count": 9, "outputs": [], "source": [ "month_2 = pd.read_excel(os.path.join(raw_dataset_path, file_temp))" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:08:02.675539400Z", "start_time": "2024-01-24T08:07:13.301903200Z" } }, "id": "218a0b12842d39bd" }, { "cell_type": "code", "execution_count": 10, "outputs": [ { "data": { "text/plain": " DemodeIP VastIP Business BeginTime Duration \\\n0 10.1.2.14 99.100.94.209 数据 2023-02-09 18:44:44 00 00:00:10 \n1 10.1.2.17 99.100.65.129 数据 2023-02-09 18:44:44 00 00:00:10 \n2 10.1.2.17 99.100.37.209 数据 2023-02-09 18:44:44 00 00:00:10 \n3 10.1.2.17 99.100.96.33 数据 2023-02-09 18:44:49 00 00:00:10 \n4 10.1.2.17 99.100.91.1 数据 2023-02-09 18:44:45 00 00:00:15 \n... ... ... ... ... ... \n501906 10.1.2.17 99.100.106.113 数据 2023-02-28 22:24:01 00 00:00:11 \n501907 10.1.2.17 99.100.90.17 数据 2023-02-28 22:23:59 00 00:00:15 \n501908 10.1.2.17 99.100.94.17 数据 2023-02-28 22:23:51 00 00:00:24 \n501909 10.1.2.17 99.100.93.33 数据 2023-02-28 22:24:00 00 00:00:17 \n501910 10.1.2.17 99.100.106.161 数据 2023-02-28 22:24:07 00 00:00:11 \n\n StreamNum SlotType CarrierType Frequency \n0 8 64 K 1 146 220 KHz \n1 1 64 K 1 147 570 KHz \n2 1 64 K 1 147 480 KHz \n3 1 64 K 1 147 570 KHz \n4 1 64 K 1 147 570 KHz \n... ... ... ... ... \n501906 2 64 K 1 148 140 KHz \n501907 2 64 K 1 148 140 KHz \n501908 1 64 K 1 147 480 KHz \n501909 1 64 K 1 147 660 KHz \n501910 1 64 K 1 147 750 KHz \n\n[501911 rows x 9 columns]", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DemodeIPVastIPBusinessBeginTimeDurationStreamNumSlotTypeCarrierTypeFrequency
010.1.2.1499.100.94.209数据2023-02-09 18:44:4400 00:00:10864 K1 146 220 KHz
110.1.2.1799.100.65.129数据2023-02-09 18:44:4400 00:00:10164 K1 147 570 KHz
210.1.2.1799.100.37.209数据2023-02-09 18:44:4400 00:00:10164 K1 147 480 KHz
310.1.2.1799.100.96.33数据2023-02-09 18:44:4900 00:00:10164 K1 147 570 KHz
410.1.2.1799.100.91.1数据2023-02-09 18:44:4500 00:00:15164 K1 147 570 KHz
..............................
50190610.1.2.1799.100.106.113数据2023-02-28 22:24:0100 00:00:11264 K1 148 140 KHz
50190710.1.2.1799.100.90.17数据2023-02-28 22:23:5900 00:00:15264 K1 148 140 KHz
50190810.1.2.1799.100.94.17数据2023-02-28 22:23:5100 00:00:24164 K1 147 480 KHz
50190910.1.2.1799.100.93.33数据2023-02-28 22:24:0000 00:00:17164 K1 147 660 KHz
50191010.1.2.1799.100.106.161数据2023-02-28 22:24:0700 00:00:11164 K1 147 750 KHz
\n

501911 rows × 9 columns

\n
" }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "month_2" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:08:15.679497800Z", "start_time": "2024-01-24T08:08:15.625749700Z" } }, "id": "cd5e9dc00b3ea405" }, { "cell_type": "code", "execution_count": 11, "outputs": [], "source": [ "user_dict = month_2['VastIP'].unique()" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:08:53.695050100Z", "start_time": "2024-01-24T08:08:53.658025800Z" } }, "id": "98c5135e5a79dfb9" }, { "cell_type": "code", "execution_count": 12, "outputs": [ { "data": { "text/plain": "array(['99.100.94.209', '99.100.65.129', '99.100.37.209', '99.100.96.33',\n '99.100.91.1', '99.100.103.161', '99.100.104.193', '99.100.104.65',\n '99.100.34.129', '99.100.38.225', '99.100.106.33',\n '99.100.106.145', '99.100.99.241', '99.100.97.225', '99.100.37.33',\n '99.100.102.161', '99.100.88.225', '99.100.93.129',\n '99.100.39.129', '99.100.37.161', '99.100.100.225',\n '99.100.101.241', '99.100.93.161', '99.100.97.129', '99.100.2.81',\n '99.100.104.49', '99.100.103.145', '99.100.104.161',\n '99.100.98.97', '99.100.94.17', '99.100.90.225', '99.100.101.81',\n '99.100.99.17', '99.100.38.49', '99.100.105.193', '99.100.103.225',\n '99.100.91.33', '99.100.93.33', '99.100.106.209', '99.100.105.177',\n '99.100.102.177', '99.100.39.81', '99.100.67.161', '99.100.92.65',\n '99.100.92.1', '99.100.106.161', '99.100.105.145',\n '99.100.105.225', '99.100.88.65', '99.100.90.1', '99.100.70.241',\n '99.100.66.145', '99.100.66.97', '99.100.69.161', '99.100.97.81',\n '99.100.106.1', '99.100.91.65', '99.100.91.177', '99.100.89.97',\n '99.100.38.1', '99.100.39.49', '99.100.103.193', '99.100.95.177',\n '99.100.106.113', '99.100.37.113', '99.100.65.113',\n '99.100.106.177', '99.100.89.81', '99.100.94.161', '99.100.88.49',\n '99.100.36.145', '99.100.96.1', '99.100.2.193', '99.100.98.49',\n '99.100.103.241', '99.100.103.17', '99.100.101.1', '99.100.97.33',\n '99.100.93.145', '99.100.35.161', '99.100.104.241',\n '99.100.70.225', '99.100.90.209', '99.100.98.81', '99.100.91.97',\n '99.100.95.81', '99.100.105.65', '99.100.39.97', '99.100.2.241',\n '99.100.36.33', '99.100.93.113', '99.100.104.1', '99.100.95.193',\n '99.100.102.193', '99.100.101.177', '99.100.66.209',\n '99.100.90.81', '99.100.68.209', '99.100.69.145', '99.100.87.1',\n '99.100.97.193', '99.100.101.161', '99.100.68.177', '99.100.94.81',\n '99.100.97.161', '99.100.92.113', '99.100.88.161', '99.100.92.145',\n '99.100.70.129', '99.100.94.241', '99.100.87.161', '99.100.92.161',\n '99.100.89.17', '99.100.67.81', '99.100.65.209', '99.100.92.129',\n '99.100.100.161', '99.100.90.65', '99.100.106.17', '99.100.96.81',\n '99.100.95.161', '99.100.67.129', '99.100.70.97', '99.100.34.161',\n '99.100.86.241', '99.100.88.17', '99.100.38.161', '99.100.67.193',\n '99.100.67.97', '99.100.104.81', '99.100.90.177', '99.100.98.225',\n '99.100.103.209', '99.100.2.209', '99.100.70.113', '99.100.92.33',\n '99.100.106.49', '99.100.100.209', '99.100.97.65', '99.100.105.81',\n '99.100.90.241', '99.100.37.145', '99.100.88.33', '99.100.101.145',\n '99.100.38.177', '99.100.90.17', '99.100.93.241', '99.100.88.241',\n '99.100.99.49', '99.100.69.129', '99.100.96.17', '99.100.66.193',\n '99.100.96.49', '99.100.67.113', '99.100.102.225', '99.100.96.65',\n '99.100.101.65', '99.100.88.81', '99.100.89.145', '99.100.89.241',\n '99.100.94.97', '99.100.94.113', '99.100.89.129', '99.100.110.129',\n '99.100.104.33', '99.100.106.81', '99.100.109.161',\n '99.100.105.33', '99.100.98.145', '99.100.95.241', '99.100.110.97',\n '99.100.110.209', '99.100.102.145', '99.100.102.81',\n '99.100.89.113', '99.100.96.209', '99.100.106.225', '99.100.95.49',\n '99.100.34.209', '99.100.65.145', '99.100.93.49', '99.100.90.113',\n '99.100.100.145', '99.100.37.97', '99.100.107.65',\n '99.100.114.113', '99.100.66.129', '99.100.67.241', '99.100.69.33',\n '99.100.39.33', '99.100.104.177', '99.100.91.209', '99.100.68.129',\n '99.100.103.65', '99.100.99.65', '99.100.94.225', '99.100.95.17',\n '99.100.91.49', '99.100.34.225', '99.100.98.1', '99.100.91.81',\n '99.100.103.177', '99.100.101.225', '99.100.96.129',\n '99.100.103.49', '99.100.67.33', '99.100.103.113', '99.100.99.177',\n '99.100.91.145', '99.100.104.145', '99.100.109.193',\n '99.100.65.225', '99.100.98.129', '99.100.66.33', '99.100.35.209',\n '99.100.115.177', '99.100.96.145', '99.100.98.177',\n '99.100.69.225', '99.100.67.65', '99.100.103.33', '99.100.105.241',\n '99.100.69.97', '99.100.71.33', '99.100.65.65', '99.100.65.49',\n '99.100.89.1', '99.100.37.49', '99.100.34.241', '99.100.107.1',\n '99.100.35.177', '99.100.35.193', '99.100.70.161',\n '99.100.102.209', '99.100.39.1', '99.100.89.161', '99.100.69.209',\n '99.100.100.193', '99.100.100.241', '99.100.97.177',\n '99.100.105.161', '99.100.104.129', '99.100.108.177',\n '99.100.87.17', '99.100.109.65', '99.100.94.1', '99.100.95.225',\n '99.100.97.209', '99.100.104.113', '99.100.68.81', '99.100.35.33',\n '99.100.94.177', '99.100.105.97', '99.100.112.145',\n '99.100.105.17', '99.100.113.17', '99.100.94.33', '99.100.64.81',\n '99.100.111.177', '99.100.35.129', '99.100.113.33', '99.100.114.1',\n '99.100.107.49', '99.100.115.145', '99.100.99.129',\n '99.100.109.241', '99.100.97.97', '99.100.66.113', '99.100.71.81',\n '99.100.107.241', '99.100.64.193', '99.100.87.33',\n '99.100.116.113', '99.100.37.65', '99.100.38.33', '99.100.110.33',\n '99.100.71.17', '99.100.105.49', '99.100.88.145', '99.100.94.129',\n '99.100.106.97', '99.100.17.161', '99.100.24.49', '99.100.95.113',\n '99.100.99.97', '99.100.109.49', '99.100.93.225', '99.100.34.145',\n '99.100.101.33', '99.100.68.33', '99.100.37.129', '99.100.104.225',\n '99.100.108.97', '99.100.114.225', '99.100.38.241',\n '99.100.98.113', '99.100.111.145', '99.100.110.1', '99.100.110.17',\n '99.100.91.17', '99.100.132.209', '99.100.69.17', '99.100.101.97',\n '99.100.92.81', '99.100.93.209', '99.100.36.241', '99.100.92.49',\n '99.100.67.225', '99.100.93.17', '99.100.70.1', '99.100.70.193',\n '99.100.38.97'], dtype=object)" }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "user_dict" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:08:57.350387600Z", "start_time": "2024-01-24T08:08:57.296165300Z" } }, "id": "f1408bccf6d0da33" }, { "cell_type": "code", "execution_count": 13, "outputs": [], "source": [ "file_temp = file_list[1]" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:14:22.900688600Z", "start_time": "2024-01-24T08:14:22.857392Z" } }, "id": "359021a931cb4bd5" }, { "cell_type": "code", "execution_count": 14, "outputs": [], "source": [ "month_3 = pd.read_excel(os.path.join(raw_dataset_path, file_temp))" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:16:55.615824600Z", "start_time": "2024-01-24T08:15:12.920108100Z" } }, "id": "e51dcae677c978ab" }, { "cell_type": "code", "execution_count": 15, "outputs": [], "source": [ "user_dict_month3 = month_3.VastIP.unique()" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:17:48.624764100Z", "start_time": "2024-01-24T08:17:48.591291200Z" } }, "id": "9fa787e87d5a6631" }, { "cell_type": "code", "execution_count": 16, "outputs": [ { "data": { "text/plain": "array(['99.100.106.161', '99.100.100.209', '99.100.92.65', '99.100.93.33',\n '99.100.103.225', '99.100.90.17', '99.100.103.161', '99.100.94.17',\n '99.100.93.241', '99.100.93.145', '99.100.95.81', '99.100.92.1',\n '99.100.67.65', '99.100.70.225', '99.100.93.49', '99.100.105.177',\n '99.100.106.113', '99.100.106.33', '99.100.36.33',\n '99.100.103.209', '99.100.39.129', '99.100.93.129', '99.100.39.33',\n '99.100.38.49', '99.100.106.209', '99.100.100.225',\n '99.100.105.145', '99.100.93.161', '99.100.97.33', '99.100.90.81',\n '99.100.101.177', '99.100.92.161', '99.100.37.33', '99.100.96.33',\n '99.100.66.145', '99.100.98.49', '99.100.103.145', '99.100.90.1',\n '99.100.102.193', '99.100.104.49', '99.100.95.177', '99.100.67.97',\n '99.100.106.1', '99.100.96.209', '99.100.99.241', '99.100.104.1',\n '99.100.104.33', '99.100.102.161', '99.100.39.81',\n '99.100.102.177', '99.100.101.161', '99.100.37.113',\n '99.100.103.241', '99.100.89.81', '99.100.94.161', '99.100.95.193',\n '99.100.106.145', '99.100.106.177', '99.100.97.193',\n '99.100.91.49', '99.100.97.225', '99.100.97.177', '99.100.68.209',\n '99.100.104.241', '99.100.39.97', '99.100.98.177', '99.100.88.81',\n '99.100.67.113', '99.100.98.97', '99.100.97.129', '99.100.100.241',\n '99.100.89.17', '99.100.93.113', '99.100.69.161', '99.100.105.193',\n '99.100.94.225', '99.100.97.209', '99.100.68.177', '99.100.90.241',\n '99.100.89.145', '99.100.105.225', '99.100.66.33', '99.100.87.1',\n '99.100.98.145', '99.100.91.65', '99.100.101.1', '99.100.106.17',\n '99.100.37.161', '99.100.70.241', '99.100.90.209', '99.100.91.177',\n '99.100.97.161', '99.100.66.209', '99.100.105.65', '99.100.92.145',\n '99.100.88.161', '99.100.70.129', '99.100.99.65', '99.100.88.225',\n '99.100.95.17', '99.100.90.113', '99.100.89.97', '99.100.67.81',\n '99.100.90.225', '99.100.35.161', '99.100.65.209', '99.100.91.33',\n '99.100.67.33', '99.100.88.65', '99.100.99.17', '99.100.99.177',\n '99.100.100.161', '99.100.65.129', '99.100.38.161',\n '99.100.37.209', '99.100.67.129', '99.100.98.225', '99.100.70.193',\n '99.100.86.241', '99.100.95.161', '99.100.70.97', '99.100.35.209',\n '99.100.90.177', '99.100.94.177', '99.100.92.33', '99.100.88.17',\n '99.100.38.225', '99.100.67.225', '99.100.71.33', '99.100.67.193',\n '99.100.104.81', '99.100.95.225', '99.100.105.81', '99.100.88.33',\n '99.100.2.209', '99.100.34.161', '99.100.2.241', '99.100.36.145',\n '99.100.97.81', '99.100.2.81', '99.100.104.161', '99.100.103.17',\n '99.100.104.113', '99.100.88.241', '99.100.66.97', '99.100.103.49',\n '99.100.103.193', '99.100.104.145', '99.100.96.49', '99.100.2.193',\n '99.100.98.1', '99.100.102.225', '99.100.66.193', '99.100.88.145',\n '99.100.101.81', '99.100.104.193', '99.100.94.129',\n '99.100.65.145', '99.100.89.161', '99.100.71.17', '99.100.101.241',\n '99.100.92.49', '99.100.101.225', '99.100.106.81', '99.100.103.33',\n '99.100.104.225', '99.100.99.97', '99.100.94.241', '99.100.92.113',\n '99.100.115.129', '99.100.115.17', '99.100.95.241', '99.100.37.65',\n '99.100.87.17', '99.100.88.177', '99.100.105.49', '99.100.94.1',\n '99.100.105.17', '99.100.89.241', '99.100.68.1', '99.100.96.65',\n '99.100.113.113', '99.100.100.177', '99.100.94.209', '99.100.67.1',\n '99.100.71.81', '99.100.96.81', '99.100.104.177', '99.100.101.65',\n '99.100.91.1', '99.100.38.241', '99.100.114.113', '99.100.105.33',\n '99.100.67.145', '99.100.102.145', '99.100.114.1',\n '99.100.102.129', '99.100.99.81', '99.100.11.97', '99.100.105.1',\n '99.100.105.129', '99.100.68.129', '99.100.102.81',\n '99.100.102.209', '99.100.96.129', '99.100.91.81', '99.100.93.17',\n '99.100.70.1', '99.100.106.49', '99.100.99.49', '99.100.114.33',\n '99.100.69.193', '99.100.65.65', '99.100.65.49', '99.100.39.49',\n '99.100.101.97', '99.100.36.193', '99.100.87.161', '99.100.102.1',\n '99.100.91.209', '99.100.111.33', '99.100.97.97', '99.100.97.65',\n '99.100.88.49', '99.100.108.177', '99.100.95.1', '99.100.35.193',\n '99.100.108.97', '99.100.35.97', '99.100.106.97', '99.100.87.33',\n '99.100.69.145', '99.100.99.225', '99.100.98.129', '99.100.35.33',\n '99.100.34.145', '99.100.101.33', '99.100.68.33', '99.100.17.161',\n '99.100.35.177', '99.100.37.49', '99.100.92.225', '99.100.38.129',\n '99.100.96.17', '99.100.30.225', '99.100.31.177', '99.100.35.1',\n '99.100.35.65', '99.100.93.65', '99.100.19.33', '99.100.115.145',\n '99.100.67.161', '99.100.36.113', '99.100.34.225', '99.100.69.97',\n '99.100.107.1', '99.100.96.225', '99.100.109.193',\n '99.100.113.193', '99.100.115.177', '99.100.106.241',\n '99.100.87.49', '99.100.38.209', '99.100.112.97', '99.100.113.33',\n '99.100.67.49', '99.100.116.33', '99.100.107.33', '99.100.71.1',\n '99.100.39.17', '99.100.35.145', '99.100.70.145', '99.100.110.97',\n '99.100.70.17', '99.100.109.49', '99.100.67.241', '99.100.71.129',\n '99.100.109.65', '99.100.113.17', '99.100.67.17', '99.100.89.1',\n '99.100.114.225', '99.100.114.49', '99.100.103.177',\n '99.100.110.145', '99.100.97.113', '99.100.115.193',\n '99.100.91.17', '99.100.91.31', '99.100.105.241', '99.100.68.81',\n '99.100.38.177', '99.100.108.1', '99.100.104.129', '99.100.90.97',\n '99.100.114.81', '99.100.90.145', '99.100.112.145',\n '99.100.96.145', '99.100.105.97', '99.100.65.113', '99.100.108.49',\n '99.100.65.241', '99.100.111.177', '99.100.102.33',\n '99.100.97.145', '99.100.99.161', '99.100.38.1', '99.100.116.241',\n '99.100.95.113', '99.100.64.17', '99.100.132.193', '99.100.37.145',\n '99.100.90.65', '99.100.70.113', '99.100.94.113', '99.100.88.193',\n '99.100.91.161', '99.100.93.177', '99.100.34.129', '99.100.104.65',\n '99.100.89.129', '99.100.97.49', '99.100.98.113', '99.100.103.113',\n '99.100.103.11', '99.100.103.65', '99.100.98.81', '99.100.34.113',\n '99.100.112.129', '99.100.97.241', '99.100.99.129', '99.100.37.97',\n '99.100.104.209', '99.100.66.129', '99.100.4.177',\n '99.100.110.129', '99.100.38.81', '99.100.93.209', '99.100.36.241',\n '99.100.34.241', '99.100.24.225', '99.100.90.129', '99.100.17.145',\n '99.100.34.209', '99.100.112.81', '99.100.88.209', '99.100.97.17',\n '99.100.71.65', '99.100.108.17', '99.100.71.113', '99.100.65.33',\n '99.100.66.225', '99.100.110.209', '99.100.38.33', '99.100.66.49',\n '99.100.99.145', '99.100.107.17', '99.100.70.161', '99.100.87.65',\n '99.100.106.129', '99.100.109.161', '99.100.101.129',\n '99.100.7.17', '99.100.103.1', '99.100.100.193', '99.100.69.1',\n '99.100.68.145', '99.100.99.209', '99.100.39.65', '99.100.66.161',\n '99.100.107.65', '99.100.69.33', '99.100.65.225', '99.100.88.97',\n '99.100.90.49', '99.100.92.97', '99.100.96.1', '99.100.91.97',\n '99.100.89.113', '99.100.106.225', '99.100.68.193',\n '99.100.36.129', '99.100.69.209', '99.100.66.177', '99.100.96.241',\n '99.100.115.241', '99.100.95.97', '99.100.39.1', '99.100.67.209',\n '99.100.111.145', '99.100.69.129', '99.100.110.33', '99.100.94.81',\n '99.100.70.209', '99.100.107.49', '99.100.68.49', '99.100.68.65',\n '99.100.37.1', '99.100.69.177', '99.100.65.97', '99.100.105.161',\n '99.100.98.65', '99.100.64.193', '99.100.64.145', '99.100.87.129',\n '99.100.92.17', '99.100.64.225', '99.100.91.113', '99.100.92.177',\n '99.100.68.161', '99.100.90.193', '99.100.87.81', '99.100.36.17',\n '99.100.32.1', '99.100.31.225', '99.100.64.113', '99.100.115.1',\n '99.100.37.81', '99.100.31.33', '99.100.38.113', '99.100.70.49',\n '99.100.5.209', '99.100.30.161', '99.100.109.129',\n '99.100.107.145', '99.100.109.177', '99.100.92.129',\n '99.100.23.129', '99.100.23.209'], dtype=object)" }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "user_dict_month3" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:17:51.929728100Z", "start_time": "2024-01-24T08:17:51.902868300Z" } }, "id": "22bf9bede244b747" }, { "cell_type": "code", "execution_count": 17, "outputs": [ { "ename": "ValueError", "evalue": "operands could not be broadcast together with shapes (312,) (430,) ", "output_type": "error", "traceback": [ "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m", "\u001B[1;31mValueError\u001B[0m Traceback (most recent call last)", "\u001B[1;32m\u001B[0m in \u001B[0;36m\u001B[1;34m\u001B[0m\n\u001B[1;32m----> 1\u001B[1;33m \u001B[0muser_dict\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0muser_dict_month3\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m", "\u001B[1;31mValueError\u001B[0m: operands could not be broadcast together with shapes (312,) (430,) " ] } ], "source": [ "user_dict + user_dict_month3" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:18:12.658480100Z", "start_time": "2024-01-24T08:18:12.533357Z" } }, "id": "e86bd06c06712b5a" }, { "cell_type": "code", "execution_count": 18, "outputs": [], "source": [ "user_dict_2_3 = np.union1d(user_dict_month3, user_dict)" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:20:12.078282800Z", "start_time": "2024-01-24T08:20:12.048371700Z" } }, "id": "f9d6fcbc370ac39e" }, { "cell_type": "markdown", "source": [], "metadata": { "collapsed": false }, "id": "25db5d7f0b4f8733" }, { "cell_type": "code", "execution_count": 20, "outputs": [], "source": [ "user_dict_2_3\n", "from tqdm import tqdm" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:46:30.722495600Z", "start_time": "2024-01-24T08:46:30.594081600Z" } }, "id": "f208635994d4914" }, { "cell_type": "code", "execution_count": 21, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 0/10 [00:49\u001B[0m in \u001B[0;36m\u001B[1;34m\u001B[0m\n\u001B[0;32m 4\u001B[0m \u001B[0mtp\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mpd\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mread_excel\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mos\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mpath\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mjoin\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mraw_dataset_path\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mf\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 5\u001B[0m \u001B[0muser_dict\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0munion1d\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0muser_dict\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mtp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mVastIP\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0munique\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m----> 6\u001B[1;33m \u001B[0mfreq_point\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0munion1d\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mfreq_point\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mtp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mFrequency\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0munique\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m", "\u001B[1;32m<__array_function__ internals>\u001B[0m in \u001B[0;36munion1d\u001B[1;34m(*args, **kwargs)\u001B[0m\n", "\u001B[1;32mD:\\Apps\\Anaconda3\\lib\\site-packages\\numpy\\lib\\arraysetops.py\u001B[0m in \u001B[0;36munion1d\u001B[1;34m(ar1, ar2)\u001B[0m\n\u001B[0;32m 747\u001B[0m \u001B[0marray\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;36m1\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;36m2\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;36m3\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;36m4\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;36m6\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 748\u001B[0m \"\"\"\n\u001B[1;32m--> 749\u001B[1;33m \u001B[1;32mreturn\u001B[0m \u001B[0munique\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mconcatenate\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mar1\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mar2\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0maxis\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;32mNone\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m 750\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 751\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n", "\u001B[1;32m<__array_function__ internals>\u001B[0m in \u001B[0;36munique\u001B[1;34m(*args, **kwargs)\u001B[0m\n", "\u001B[1;32mD:\\Apps\\Anaconda3\\lib\\site-packages\\numpy\\lib\\arraysetops.py\u001B[0m in \u001B[0;36munique\u001B[1;34m(ar, return_index, return_inverse, return_counts, axis)\u001B[0m\n\u001B[0;32m 259\u001B[0m \u001B[0mar\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0masanyarray\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mar\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 260\u001B[0m \u001B[1;32mif\u001B[0m \u001B[0maxis\u001B[0m \u001B[1;32mis\u001B[0m \u001B[1;32mNone\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 261\u001B[1;33m \u001B[0mret\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0m_unique1d\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mar\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mreturn_index\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mreturn_inverse\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mreturn_counts\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m 262\u001B[0m \u001B[1;32mreturn\u001B[0m \u001B[0m_unpack_tuple\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mret\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 263\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n", "\u001B[1;32mD:\\Apps\\Anaconda3\\lib\\site-packages\\numpy\\lib\\arraysetops.py\u001B[0m in \u001B[0;36m_unique1d\u001B[1;34m(ar, return_index, return_inverse, return_counts)\u001B[0m\n\u001B[0;32m 320\u001B[0m \u001B[0maux\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mar\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mperm\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 321\u001B[0m \u001B[1;32melse\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 322\u001B[1;33m \u001B[0mar\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0msort\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m 323\u001B[0m \u001B[0maux\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mar\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 324\u001B[0m \u001B[0mmask\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mempty\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0maux\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mshape\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdtype\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mbool_\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n", "\u001B[1;31mTypeError\u001B[0m: '<' not supported between instances of 'str' and 'float'" ] } ], "source": [ "user_dict = []\n", "freq_point\n", "for f in tqdm(file_list):\n", " tp = pd.read_excel(os.path.join(raw_dataset_path, f))\n", " user_dict = np.union1d(user_dict, tp.VastIP.unique())\n", " freq_point = np.union1d(freq_point, tp.Frequency.unique())" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:48:49.422148500Z", "start_time": "2024-01-24T08:47:59.874651800Z" } }, "id": "5a75925d095fd2a7" }, { "cell_type": "code", "execution_count": 22, "outputs": [ { "data": { "text/plain": "array(['1 146 220 KHz', '1 147 570 KHz', '1 147 480 KHz', '1 147 660 KHz',\n '1 146 130 KHz', '1 148 140 KHz', '1 147 750 KHz', '1 146 400 KHz',\n '1 147 840 KHz', '1 149 310 KHz', '1 146 040 KHz', '1 146 310 KHz',\n '1 147 930 KHz', '1 149 400 KHz', '1 148 770 KHz', '1 146 490 KHz',\n '1 146 670 KHz', '1 148 860 KHz', '1 146 580 KHz', '1 147 120 KHz',\n '1 147 300 KHz', '1 148 230 KHz', '1 148 950 KHz', '1 146 760 KHz',\n '1 079 720 KHz', '1 146 850 KHz', '1 146 940 KHz', '1 147 030 KHz',\n '1 147 390 KHz', '1 078 930 KHz', '1 149 040 KHz', '1 147 210 KHz',\n nan, '1 079 540 KHz', '1 078 580 KHz', '1 079 360 KHz',\n '1 079 180 KHz', '1 143 560 KHz', '1 144 930 KHz', '1 077 040 KHz'],\n dtype=object)" }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "month_2.Frequency.unique()" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:49:58.297198300Z", "start_time": "2024-01-24T08:49:58.208071300Z" } }, "id": "7468058e6858efaf" }, { "cell_type": "code", "execution_count": 23, "outputs": [ { "ename": "TypeError", "evalue": "'<' not supported between instances of 'str' and 'float'", "output_type": "error", "traceback": [ "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m", "\u001B[1;31mTypeError\u001B[0m Traceback (most recent call last)", "\u001B[1;32m\u001B[0m in \u001B[0;36m\u001B[1;34m\u001B[0m\n\u001B[1;32m----> 1\u001B[1;33m \u001B[0mfreq_point\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0munion1d\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mfreq_point\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mmonth_2\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mFrequency\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0munique\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m", "\u001B[1;32m<__array_function__ internals>\u001B[0m in \u001B[0;36munion1d\u001B[1;34m(*args, **kwargs)\u001B[0m\n", "\u001B[1;32mD:\\Apps\\Anaconda3\\lib\\site-packages\\numpy\\lib\\arraysetops.py\u001B[0m in \u001B[0;36munion1d\u001B[1;34m(ar1, ar2)\u001B[0m\n\u001B[0;32m 747\u001B[0m \u001B[0marray\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;36m1\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;36m2\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;36m3\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;36m4\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;36m6\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 748\u001B[0m \"\"\"\n\u001B[1;32m--> 749\u001B[1;33m \u001B[1;32mreturn\u001B[0m \u001B[0munique\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mconcatenate\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mar1\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mar2\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0maxis\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;32mNone\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m 750\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 751\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n", "\u001B[1;32m<__array_function__ internals>\u001B[0m in \u001B[0;36munique\u001B[1;34m(*args, **kwargs)\u001B[0m\n", "\u001B[1;32mD:\\Apps\\Anaconda3\\lib\\site-packages\\numpy\\lib\\arraysetops.py\u001B[0m in \u001B[0;36munique\u001B[1;34m(ar, return_index, return_inverse, return_counts, axis)\u001B[0m\n\u001B[0;32m 259\u001B[0m \u001B[0mar\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0masanyarray\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mar\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 260\u001B[0m \u001B[1;32mif\u001B[0m \u001B[0maxis\u001B[0m \u001B[1;32mis\u001B[0m \u001B[1;32mNone\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 261\u001B[1;33m \u001B[0mret\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0m_unique1d\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mar\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mreturn_index\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mreturn_inverse\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mreturn_counts\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m 262\u001B[0m \u001B[1;32mreturn\u001B[0m \u001B[0m_unpack_tuple\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mret\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 263\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n", "\u001B[1;32mD:\\Apps\\Anaconda3\\lib\\site-packages\\numpy\\lib\\arraysetops.py\u001B[0m in \u001B[0;36m_unique1d\u001B[1;34m(ar, return_index, return_inverse, return_counts)\u001B[0m\n\u001B[0;32m 320\u001B[0m \u001B[0maux\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mar\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mperm\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 321\u001B[0m \u001B[1;32melse\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 322\u001B[1;33m \u001B[0mar\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0msort\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m 323\u001B[0m \u001B[0maux\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mar\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 324\u001B[0m \u001B[0mmask\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mempty\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0maux\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mshape\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdtype\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mbool_\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n", "\u001B[1;31mTypeError\u001B[0m: '<' not supported between instances of 'str' and 'float'" ] } ], "source": [ "freq_point = np.union1d(freq_point, month_2.Frequency.unique())" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:50:31.422526800Z", "start_time": "2024-01-24T08:50:31.249467100Z" } }, "id": "1ef93050f810cbf7" }, { "cell_type": "markdown", "source": [], "metadata": { "collapsed": false }, "id": "3c2d8d1392126a87" }, { "cell_type": "code", "execution_count": 29, "outputs": [], "source": [ "tt = month_2['Frequency']" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:54:41.090942500Z", "start_time": "2024-01-24T08:54:40.985068400Z" } }, "id": "cf856df5660045c9" }, { "cell_type": "code", "execution_count": 30, "outputs": [ { "data": { "text/plain": "0 1 146 220 KHz\n1 1 147 570 KHz\n2 1 147 480 KHz\n3 1 147 570 KHz\n4 1 147 570 KHz\n ... \n501906 1 148 140 KHz\n501907 1 148 140 KHz\n501908 1 147 480 KHz\n501909 1 147 660 KHz\n501910 1 147 750 KHz\nName: Frequency, Length: 501911, dtype: object" }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tt" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T08:54:42.335759900Z", "start_time": "2024-01-24T08:54:42.236689100Z" } }, "id": "d8cf6daf86319783" }, { "cell_type": "code", "execution_count": 31, "outputs": [ { "data": { "text/plain": "array(['1 146 220 KHz', '1 147 570 KHz', '1 147 480 KHz', '1 147 660 KHz',\n '1 146 130 KHz', '1 148 140 KHz', '1 147 750 KHz', '1 146 400 KHz',\n '1 147 840 KHz', '1 149 310 KHz', '1 146 040 KHz', '1 146 310 KHz',\n '1 147 930 KHz', '1 149 400 KHz', '1 148 770 KHz', '1 146 490 KHz',\n '1 146 670 KHz', '1 148 860 KHz', '1 146 580 KHz', '1 147 120 KHz',\n '1 147 300 KHz', '1 148 230 KHz', '1 148 950 KHz', '1 146 760 KHz',\n '1 079 720 KHz', '1 146 850 KHz', '1 146 940 KHz', '1 147 030 KHz',\n '1 147 390 KHz', '1 078 930 KHz', '1 149 040 KHz', '1 147 210 KHz',\n 'nan', '1 079 540 KHz', '1 078 580 KHz', '1 079 360 KHz',\n '1 079 180 KHz', '1 143 560 KHz', '1 144 930 KHz', '1 077 040 KHz'],\n dtype='\u001B[0m in \u001B[0;36m\u001B[1;34m\u001B[0m\n\u001B[0;32m 1\u001B[0m \u001B[1;32mfor\u001B[0m \u001B[0mf\u001B[0m \u001B[1;32min\u001B[0m \u001B[0mtqdm\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mfile_list\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m----> 2\u001B[1;33m \u001B[0mtp\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mpd\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mread_excel\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mos\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mpath\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mjoin\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mraw_dataset_path\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mf\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mread_only\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;32mTrue\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m 3\u001B[0m \u001B[0mtp\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;34m'user'\u001B[0m\u001B[1;33m]\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mtp\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;34m'VastIP'\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mmap\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0muser_dict_code\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 4\u001B[0m \u001B[0mtp\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;34m'type'\u001B[0m\u001B[1;33m]\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mtp\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;34m'Business'\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mmap\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mtype_dict\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 5\u001B[0m \u001B[0mtp\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;34m'freq'\u001B[0m\u001B[1;33m]\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mtp\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;34m'Frequency'\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mmap\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mfreq_point_code\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n", "\u001B[1;32mD:\\Apps\\Anaconda3\\lib\\site-packages\\pandas\\util\\_decorators.py\u001B[0m in \u001B[0;36mwrapper\u001B[1;34m(*args, **kwargs)\u001B[0m\n\u001B[0;32m 297\u001B[0m )\n\u001B[0;32m 298\u001B[0m \u001B[0mwarnings\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mwarn\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mmsg\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mFutureWarning\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mstacklevel\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mstacklevel\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 299\u001B[1;33m \u001B[1;32mreturn\u001B[0m \u001B[0mfunc\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m*\u001B[0m\u001B[0margs\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;33m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m 300\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 301\u001B[0m \u001B[1;32mreturn\u001B[0m \u001B[0mwrapper\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n", "\u001B[1;31mTypeError\u001B[0m: read_excel() got an unexpected keyword argument 'read_only'" ] } ], "source": [ "for f in tqdm(file_list):\n", " tp = read_excel(os.path.join(raw_dataset_path, f))\n", " tp['user'] = tp['VastIP'].map(user_dict_code)\n", " tp['type'] = tp['Business'].map(type_dict)\n", " tp['freq'] = tp['Frequency'].map(freq_point_code)\n", " output = os.path.join(os.path.abspath(dataset_path), f)\n", " tp[['BeginTime', 'StreamNum', 'SlotType', 'CarrierType', 'user', 'type', 'freq', 'Duration']].to_csv(output)" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T10:10:19.189405800Z", "start_time": "2024-01-24T10:10:19.078568900Z" } }, "id": "535d838c3c5a6789" }, { "cell_type": "code", "execution_count": 58, "outputs": [ { "ename": "TypeError", "evalue": "read_excel() got an unexpected keyword argument 'read_only'", "output_type": "error", "traceback": [ "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m", "\u001B[1;31mTypeError\u001B[0m Traceback (most recent call last)", "\u001B[1;32m\u001B[0m in \u001B[0;36m\u001B[1;34m\u001B[0m\n\u001B[1;32m----> 1\u001B[1;33m \u001B[0mmonth_3\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mpd\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mread_excel\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mos\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mpath\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mjoin\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mraw_dataset_path\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mfile_temp\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mengine\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;34m\"openpyxl\"\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mread_only\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;32mTrue\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m", "\u001B[1;32mD:\\Apps\\Anaconda3\\lib\\site-packages\\pandas\\util\\_decorators.py\u001B[0m in \u001B[0;36mwrapper\u001B[1;34m(*args, **kwargs)\u001B[0m\n\u001B[0;32m 297\u001B[0m )\n\u001B[0;32m 298\u001B[0m \u001B[0mwarnings\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mwarn\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mmsg\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mFutureWarning\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mstacklevel\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mstacklevel\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 299\u001B[1;33m \u001B[1;32mreturn\u001B[0m \u001B[0mfunc\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m*\u001B[0m\u001B[0margs\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;33m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m 300\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m 301\u001B[0m \u001B[1;32mreturn\u001B[0m \u001B[0mwrapper\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n", "\u001B[1;31mTypeError\u001B[0m: read_excel() got an unexpected keyword argument 'read_only'" ] } ], "source": [ "month_3 = pd.read_excel(os.path.join(raw_dataset_path, file_temp), engine=\"openpyxl\", read_only=True)" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T10:22:45.160496500Z", "start_time": "2024-01-24T10:22:45.061818900Z" } }, "id": "e37b0ee51c81f6b8" }, { "cell_type": "code", "execution_count": 60, "outputs": [], "source": [ "from openpyxl import load_workbook\n", "\n", "work_book = load_workbook(filename=\"raw_dataset\\\\Channel_03.xlsx\", read_only=True)" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T10:28:30.248658100Z", "start_time": "2024-01-24T10:28:15.850166200Z" } }, "id": "679d082e4bb85b62" }, { "cell_type": "code", "execution_count": 61, "outputs": [], "source": [ "sheet = work_book.active" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T10:29:22.430465300Z", "start_time": "2024-01-24T10:29:22.250192700Z" } }, "id": "c4e50e8c88388da4" }, { "cell_type": "code", "execution_count": 62, "outputs": [], "source": [ "df = pd.DataFrame(sheet.values)" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T10:30:48.647266600Z", "start_time": "2024-01-24T10:29:38.751286Z" } }, "id": "cd8b23dfd5ee7488" }, { "cell_type": "code", "execution_count": 65, "outputs": [], "source": [ "def read_excel(s):\n", " ff = load_workbook(filename=s, read_only=True)\n", " return pd.DataFrame(ff.active.values)" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T10:32:17.491830700Z", "start_time": "2024-01-24T10:32:17.390355600Z" } }, "id": "6ca92cf5801829b8" }, { "cell_type": "code", "execution_count": 67, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 10/10 [16:21<00:00, 98.12s/it]\n" ] } ], "source": [ "for f in tqdm(file_list):\n", " tp = pd.read_excel(os.path.join(raw_dataset_path, f))\n", " tp['user'] = tp['VastIP'].map(user_dict_code)\n", " tp['type'] = tp['Business'].map(type_dict)\n", " tp['freq'] = tp['Frequency'].map(freq_point_code)\n", " output = os.path.join(os.path.abspath(dataset_path), f)\n", " tp[['BeginTime', 'StreamNum', 'SlotType', 'CarrierType', 'user', 'type', 'freq', 'Duration']].to_csv(output)" ], "metadata": { "collapsed": false, "ExecuteTime": { "end_time": "2024-01-24T10:51:30.947246100Z", "start_time": "2024-01-24T10:35:09.672634400Z" } }, "id": "5aeb1b778e7548c9" }, { "cell_type": "code", "execution_count": null, "outputs": [], "source": [], "metadata": { "collapsed": false }, "id": "7ad1bc8eb29ddfab" } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.6" } }, "nbformat": 4, "nbformat_minor": 5 }