listing——info

start_sync_amz_RightNowRun.py

@@ -96,8 +96,8 @@ if __name__ == '__main__':
     amz_report(conn, AWS_CREDENTIALS=AWS_CREDENTIALS)
 
     # list_date = ["2023-10-22",]
-    # # list_date = [f'2023-10-{"0"+str(i) if len(str(i))==1 else i}' for i in range(23,32)]
-    # # print(list_date)
+    # list_date = [f'2023-11-{"0"+str(i) if len(str(i))==1 else i}' for i in range(27,30)]
+    # print(list_date)
     # for date_ in list_date:
     #     print(date_)
     #     print(date_.replace("-",""))

sync_amz_data/public/sp_api_client.py

@@ -20,62 +20,39 @@ import pymysql
 
 
 class SpApiRequest:
-    def __init__(self, credentials, marketplace,profile_id):
+
+    def __init__(self, credentials,marketplace):
         self.credentials = credentials
         self.marketplace = marketplace
-        self.shopInfo = shop_infos(profile_id)
-        self.timezone = self.shopInfo['time_zone']
-        self.profileid = profile_id
+        # self.shopInfo = shop_infos('3006125408623189')
+        # self.timezone = self.shopInfo['time_zone']
+        # self.profileid = '3006125408623189'
+
+    @classmethod
+    def mysql_connect_auth(cls):
+        conn = pymysql.connect(user="admin",
+        password="NSYbBSPbkGQUbOSNOeyy",
+        host="retail-data.cnrgrbcygoap.us-east-1.rds.amazonaws.com",
+        database="ansjer_dvadmin",
+        port=3306)
+        return conn
 
     def mysql_connect(self):
         conn = pymysql.connect(user="huangyifan",
-        password="123456",
-        host="192.168.1.18",
-        database="amz_sp_api",
-        port=3306)
+                               password="123456",
+                               host="192.168.1.18",
+                               database="amz_sp_api",
+                               port=3306)
         return conn
 
-    def timeToLocalTime(self,para_time=None):
-        if para_time is not None:
-            if ":" not in para_time:
-                para_time = para_time+' 0:0:0'
-            report_USTime = datetime.strptime(para_time,"%Y-%m-%d %H:%M:%S")
-        else:
-            report_USTime = datetime.now(tz=pytz.timezone(self.timezone)) + timedelta(days=-1)
-        USTime = pytz.timezone(self.timezone)
-        ustime_end = USTime.localize(datetime(report_USTime.year,report_USTime.month,report_USTime.day,23,59,59,999999))
-        ustime_start = USTime.localize(datetime(report_USTime.year,report_USTime.month,report_USTime.day,0,0,0))
-        print(self.timezone+": ","\n",ustime_start,"-",ustime_end)
-        # utctime = pytz.timezone("UTC")
-        timezoneutc = pytz.timezone('UTC')
-        startTimeutc = ustime_start.astimezone(timezoneutc)
-        endTimeutc = ustime_end.astimezone(timezoneutc)
-        formatTime = "%Y-%m-%dT%H:%M:%S+00:00"
-        startTimeutc = startTimeutc.strftime(formatTime)
-        endTimeutc = endTimeutc.strftime(formatTime)
-        print("UTC: ","\n",startTimeutc,"-",endTimeutc)
-        return startTimeutc,endTimeutc
-
-    def timeDeal(self,orgTime):
-        orgTime = parse(orgTime)
-        timezone = pytz.timezone(self.timezone)
-        shopTime = orgTime.astimezone(timezone)
-        shopTime_datetime = datetime(shopTime.year,shopTime.month,shopTime.day,shopTime.hour,shopTime.minute,shopTime.second)
-        return shopTime_datetime
-
 
-    def nowTime(self):
-        computerTimenow = datetime.now()
-        localTimeNow = datetime.now(tz=pytz.timezone(self.timezone))
-        utctimeNow = datetime.utcnow()
-        return  {"computerTimenow":computerTimenow,"localTimeNow":localTimeNow,"utctimeNow":utctimeNow}
 
     def create_report(self,**kwargs):
         reportType = kwargs['reportType']
         reportOptions =kwargs.get("reportOptions")
 
-        dataStartTime = datetime.now(tz=pytz.timezone(self.timezone)).strftime("%Y-%m-%dT%H:%M:%S") if kwargs.get("dataStartTime") is None else kwargs.get("dataStartTime")+"T00:00:00"
-        dataEndTime = datetime.now(tz=pytz.timezone(self.timezone)).strftime("%Y-%m-%dT%H:%M:%S") if kwargs.get("dataEndTime") is None else kwargs.get("dataEndTime")+"T23:59:59"
+        dataStartTime = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") if kwargs.get("dataStartTime") is None else kwargs.get("dataStartTime")+"T00:00:00"
+        dataEndTime = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") if kwargs.get("dataEndTime") is None else kwargs.get("dataEndTime")+"T23:59:59"
 
         report = Reports(credentials=self.credentials, marketplace=self.marketplace)
         rel = report.create_report(
@@ -83,7 +60,7 @@ class SpApiRequest:
             reportOptions=reportOptions,dataStartTime=dataStartTime,dataEndTime=dataEndTime
                         )
         reportId = rel.payload.get("reportId")
-        print(reportId)
+        # print(reportId)
         return reportId
 
     def decompression(self,reportId):
@@ -96,19 +73,27 @@ class SpApiRequest:
                 reportDocumentId = reportId_info.payload.get("reportDocumentId")
                 rp_table = report.get_report_document(reportDocumentId=reportDocumentId,download=False)
                 print(rp_table)
-                if rp_table.payload.get('compressionAlgorithm') is not None:
+                if rp_table.payload.get('compressionAlgorithm') is not None and self.marketplace.marketplace_id not in ['A1VC38T7YXB528']:#
                     df = pd.read_table(filepath_or_buffer=rp_table.payload['url'],compression={"method":'gzip'},encoding='iso-8859-1')
                     return df
-                else:
+                elif rp_table.payload.get('compressionAlgorithm') is not None and self.marketplace.marketplace_id in ['A1VC38T7YXB528']:
+                    df = pd.read_table(filepath_or_buffer=rp_table.payload['url'], compression={"method": 'gzip'},
+                                       encoding='Shift-JIS')
+                    # df.columns =
+                    return df
+                elif rp_table.payload.get('compressionAlgorithm') is None and self.marketplace.marketplace_id not in ['A1VC38T7YXB528']:
                     df = pd.read_table(rp_table.payload.get("url"),encoding='iso-8859-1')
                     return df
+                elif rp_table.payload.get('compressionAlgorithm') is None and self.marketplace.marketplace_id in ['A1VC38T7YXB528']:
+                    df = pd.read_table(rp_table.payload.get("url"),encoding='Shift-JIS')
+                    return df
             elif reportId_info.payload.get("processingStatus") in [ProcessingStatus.CANCELLED,ProcessingStatus.FATAL]:
                 print("取消或失败")
                 break
             time.sleep(15)
             print("please wait...")
 
-    def data_deal(self,decom_df):
+    def data_deal(self,decom_df,seller_id):
         decom_df['mainImageUrl'] = decom_df['seller-sku'].map(lambda x: self.get_mainImage_url(x))
         url_columns = [i for i in decom_df.columns if "url" in i.lower()]
         if len(url_columns) > 0:
@@ -124,17 +109,17 @@ class SpApiRequest:
         if 'quantity' in decom_df.columns:
             decom_df['quantity'] = decom_df['quantity'].map(lambda x: 0 if pd.isna(x) or np.isinf(x) else x).astype(
                 "int32")
-        decom_df['opendate_date'] = decom_df['open-date'].str.split(' ', expand=False).map(lambda x: x[0]).astype(
-            'datetime64[ns]')
+        decom_df['opendate_date'] = decom_df['open-date'].map(lambda x: self.datetime_deal(x))
         if 'add-delete' in decom_df.columns:
             decom_df['add-delete'] = decom_df['add-delete'].astype('string', errors='ignore')
         if 'will-ship-internationally' in decom_df.columns:
             decom_df['will-ship-internationally'] = decom_df['will-ship-internationally'].astype('string',errors='ignore')
         if 'expedited-shipping' in decom_df.columns:
             decom_df['expedited-shipping'] = decom_df['expedited-shipping'].astype('string',errors='ignore')
-        decom_df['updateTime'] = datetime.now(tz=pytz.timezone(self.timezone))
-        decom_df['timezone'] = self.timezone
-        decom_df['profileid'] = str(self.profileid)
+        decom_df['updateTime'] = datetime.now()
+        decom_df['timezone'] = "UTC"
+        decom_df['seller_id'] = seller_id
+        #
         decom_df['item-description'] = decom_df['item-description'].str.slice(0,500)
         decom_df[decom_df.select_dtypes(float).columns] = decom_df[decom_df.select_dtypes(float).columns].fillna(0.0)
         decom_df[decom_df.select_dtypes(int).columns] = decom_df[decom_df.select_dtypes(int).columns].fillna(0)
@@ -153,8 +138,8 @@ class SpApiRequest:
         conn = self.mysql_connect()
         print("连接成功")
         cursor = conn.cursor()
-        timezone = pytz.timezone(self.timezone)
-        bondary_date = (datetime.now(tz=timezone)).strftime("%Y-%m-%d") #+ timedelta(days=-28)
+        timezone = "UTC" #pytz.timezone(self.timezone)
+        bondary_date = (datetime.now()).strftime("%Y-%m-%d") #+ timedelta(days=-28)
         cursor.execute(f"""select * from amz_sp_api.productInfo where (mainImageUrl is not null and mainImageUrl not in ('', ' ')) and 
                         (`seller-sku` not in ('',' ') and `seller-sku` is not null) and 
                         `updateTime`>='{bondary_date}'""") #`seller-sku`,`updateTime`,`mainImageUrl`
@@ -239,29 +224,263 @@ class SpApiRequest:
             img_url = None if img is None else img.get("link")
         return img_url
 
-    def GET_FLAT_FILE_OPEN_LISTINGS_DATA(self):
-        start = time.time()
+    def datetime_deal(self,timestring):
+        timezone_ = {"AEST":"Australia/Sydney",
+                    "AEDT":"Australia/Sydney",
+                    "PST":"America/Los_Angeles",
+                    "PDT":"America/Los_Angeles",
+                    "CST":"America/Chicago",
+                    "CDT":"America/Chicago",
+                    "MET":"MET",
+                    "MEST":"MET",
+                    "BST":"Europe/London",
+                    "GMT":"GMT",
+                    "CET":"CET",
+                    "CEST":"CET",
+                    "JST":"Asia/Tokyo",
+                    "BRT":"America/Sao_Paulo"}
+
+        date_list = str.split(timestring,sep = ' ')
+        if len(date_list)<3:
+            try:
+                return datetime.strptime(date_list[0],"%Y-%m-%d")
+            except:
+                try:
+                    return datetime.strptime(date_list[0], "%Y/%m/%d")
+                except:
+                    try:
+                        return datetime.strptime(date_list[0], "%d/%m/%Y")
+                    except Exception as e:
+                        print(e)
+                        return datetime(1999, 12, 31, 0, 0, 0)
+        try:
+            time_date = datetime.strptime(date_list[0]+date_list[1],"%Y-%m-%d%H:%M:%S")
+            timezone = pytz.timezone(timezone_[date_list[2]])
+            time_ = timezone.localize(time_date)
+            return time_.astimezone(pytz.UTC)
+        except:
+            try:
+                time_date = datetime.strptime(date_list[0] + date_list[1], "%d/%m/%Y%H:%M:%S")
+                timezone = pytz.timezone(timezone_[date_list[2]])
+                time_ = timezone.localize(time_date)
+                return time_.astimezone(pytz.UTC)
+            except :
+                try:
+                    time_date = datetime.strptime(date_list[0] + date_list[1], "%Y/%m/%d%H:%M:%S")
+                    timezone = pytz.timezone(timezone_[date_list[2]])
+                    time_ = timezone.localize(time_date)
+                    return time_.astimezone(pytz.UTC)
+                except Exception as e1:
+                    print(e1)
+                    return datetime(1999,12,31,0,0,0)
+
+    def update_data(self,df,seller_id,country_code,conn):
+        cursor = conn.cursor()
+        columns = ['listing-id','seller_id',
+                              'asin1','seller-sku','country_code',
+                              'marketplace_id','quantity','fulfillment_channel',
+                              'price','opendate','status','update_datetime','product-id','product-id-type'
+                              ]
+        df_data = pd.DataFrame(columns=columns)
+        delete_list = []
+
+
+        try:
+            cursor.execute(f"""select * from
+                                ansjer_dvadmin.seller_listings where seller_id='{seller_id}' and country_code='{country_code}'""")
+            col = [i[0] for i in cursor.description]
+            query_rel = cursor.fetchall()
+            df_rel = pd.DataFrame(query_rel, columns=col)
+            df_rel['quantity'] = df_rel['quantity'].fillna(0).astype('int64')
+            df_rel['price'] = df_rel['price'].fillna(0.0).astype('float64')
+            df_rel['product_id_type'] = df_rel['product_id_type'].astype('int64')
+            df['update_datetime'] =df['update_datetime'].astype('datetime64[ns]')
+
+            df['quantity'] = df['quantity'].fillna(0).astype('int64')
+            df['price']= df['price'].fillna(0.0).astype('float64')
+            # print(df_rel.dtypes)
+            # print(df[columns].dtypes)
+            row = 0
+            while row < len(df):
+                temp_df = df.iloc[row, :]
+                listing_id = temp_df['listing-id']
+                asin = temp_df['asin1']
+                sku = temp_df['seller-sku']
+                quantity = temp_df['quantity']
+                fulfillment_channel = temp_df['fulfillment_channel']
+                price = temp_df['price']
+                product_id = temp_df['product-id']
+                temp = df_rel.query("""listing_id==@listing_id and asin==@asin and sku==@sku and quantity==@quantity and fulfillment_channel==@fulfillment_channel and price==@price and product_id==@product_id and country_code==@country_code and seller_id==@seller_id""")
+                print("需要关注数据（是否异常）：",len(temp),temp.to_numpy().tolist()) if len(temp)>1 else 1
+
+                # print(len(temp))
+                if len(temp)==0:
+                    df_data = df_data.append(temp_df,ignore_index=True)
+                    delete_list.append((seller_id,country_code,sku,listing_id,product_id))
+                row += 1
+            print("判断不同数据条数",len(delete_list))
+            print("预计更新数据条数",len(df_data))
+            try:
+                # print(tuple(delete_list))
+                if len(delete_list)>0:
+                    query = f"""delete from ansjer_dvadmin.seller_listings 
+                           where (seller_id,country_code,sku,listing_id,product_id) in %s""" #where (seller_id,country_code) in %s"""
+                    cursor.execute(query,(delete_list,))
+
+                    conn.commit()
+                    print(delete_list)
+                    print("进行中...")
+            except Exception as e:
+                print(e)
+                conn.rollback()
+            return df_data
+        except Exception as e:
+            print("错误：", e)
+            return df
+
+
+    def GET_FLAT_FILE_OPEN_LISTINGS_DATA(self,conn=None,seller_id=None):
         para = {"reportType": ReportType.GET_MERCHANT_LISTINGS_ALL_DATA}
         reportid = self.create_report(**para)
         df = self.decompression(reportid)
-        df['sellerid'] = ''
-        df['marketplace_id'] = self.marketplace.marketplace_id
-        df['country_code'] = str(self.marketplace)[-2:]
-        df['fulfillment_channel'] = df['fulfillment-channel'].map({"DEFAULT":"FBM","AMAZON_NA":"FBA"})
-
-        reserve_columns = ['listing_id','seller_id','asin','sku','country_code','marketplace_id',
-                        'quantity','fulfillment_channel','price','status']
-        print(df.columns)
-        print(df)
-
-    def GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL(self):
-        timezone_ = pytz.timezone(self.timezone)
-        shopReportday = (datetime.now(tz=timezone_) + timedelta(days=-1)).strftime("%Y-%m-%d")
+        if len(df)>0:
+            if self.marketplace.marketplace_id =='A1VC38T7YXB528':
+                df.columns = ['item-name','listing-id','seller-sku','price','quantity','open-date','product-id-type','item-description',
+                              'item-condition','overseas shipping','fast shipping','asin1','stock_number','fulfillment-channel','merchant-shipping-group','status']
+
+            df['seller_id'] = seller_id
+            df['marketplace_id'] = self.marketplace.marketplace_id
+            df['country_code'] = str(self.marketplace)[-2:]
+            if 'fulfilment-channel' in df.columns:
+                print("changed fulfilment-channel:")
+                print(seller_id,self.marketplace)
+                df['fulfillment-channel'] = df['fulfilment-channel'].copy()
+            df['fulfillment_channel'] = df['fulfillment-channel'].map(lambda x:"FBA" if not pd.isna(x) and len(x)>0 and str(x)[1:4] in "AMAZON" else x)
+            df['fulfillment_channel'] = df['fulfillment_channel'].map(lambda x: "FBM" if not pd.isna(x) and len(x)>0 and str(x)[1:4] in "DEFAULT" else x)
+
+            if 'asin1' not in df.columns:
+                df['asin1'] = ''
+            if 'product-id' not in df.columns:
+                df['product-id'] = ''
+
+            # 空值处理
+            df['quantity'] = df['quantity'].fillna(0).astype('int64',errors='ignore')
+            df[['listing-id','seller_id','asin1','seller-sku','country_code','marketplace_id','fulfillment_channel','status','product-id']] = df[['listing-id','seller_id','asin1','seller-sku','country_code','marketplace_id','fulfillment_channel','status','product-id']].fillna('').astype('string',errors='ignore')
+            df['price'] = df['price'].fillna(0.0).astype('float64',errors='ignore')
+            df.fillna('',inplace=True)
+
+            # 时间处理
+            df['opendate'] = df['open-date'].map(lambda x: self.datetime_deal(x))
+            df['update_datetime'] = datetime.now(pytz.UTC).date()
+
+            origin_columns = ['listing-id','seller_id',
+                              'asin1','seller-sku','country_code',
+                              'marketplace_id','quantity','fulfillment_channel',
+                              'price','opendate','status','update_datetime','product-id','product-id-type'
+                              ]
+            cursor = conn.cursor()
+            cursor.execute("""select product_id,asin from (select * from ansjer_dvadmin.seller_listings where asin is not null 
+                                and asin<>'' and product_id is not null and product_id <>'') t1 group by product_id,asin""")
+            query_ = cursor.fetchall()
+            col_name = [i[0] for i in cursor.description]
+            df_datatable = pd.DataFrame(query_, columns=col_name)
+            merged_df = df.merge(df_datatable[['product_id','asin']],how='left',left_on='product-id',right_on='product_id')
+            print(merged_df.head())
+            def func_(asin,asin1,product_id):
+                if 'B0' in str(product_id):
+                    return str(product_id)
+                if pd.isna(asin1) or asin1=='':
+                    return asin
+                else:
+                    return asin1
+
+            merged_df['asin1'] = merged_df.apply(lambda x:func_(x['asin'],x['asin1'],x['product-id']),axis=1) #x['asin'] if pd.isna(x['asin1']) or x['asin1']=='' else x['asin1']
+            merged_df.fillna('',inplace=True)
+            df1 = merged_df.copy()
+            print(df1[origin_columns].head(1))
+            update_df = self.update_data(df1,seller_id,str(self.marketplace)[-2:],conn)
+            # update_df.to_csv("fr.csv")
+            if len(update_df)==0:
+                return '无更新数据插入'
+
+            cursor = conn.cursor()
+
+            try:
+                insertsql = """insert into
+                ansjer_dvadmin.seller_listings(listing_id,seller_id,asin,sku,country_code,marketplace_id,quantity,
+                        fulfillment_channel,price,launch_datetime,status,update_datetime,product_id,product_id_type)
+                values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
+                conn.begin()
+                cursor.executemany(insertsql,tuple(update_df[origin_columns].to_numpy().tolist()))
+                conn.commit()
+            except Exception as e:
+                print("插入错误：",e)
+                conn.rollback()
+
+    @classmethod
+    def get_allShops(cls):
+        auth_conn = SpApiRequest.mysql_connect_auth()
+        cursor = auth_conn.cursor()
+        cursor.execute("select * from amazon_sp_report.amazon_sp_auth_info;")
+        columns_name = [i[0] for i in cursor.description]
+        rel = cursor.fetchall()
+        df = pd.DataFrame(rel, columns=columns_name)
+
+        for refresh_token in df['refresh_token'].to_numpy().tolist():
+            aws_credentials = {
+                'refresh_token': refresh_token,
+                'lwa_app_id': 'amzn1.application-oa2-client.1f9d3d4747e14b22b4b598e54e6b922e',  # 卖家中心里面开发者资料LWA凭证
+                'lwa_client_secret': 'amzn1.oa2-cs.v1.3af0f5649f5b8e151cd5bd25c10f2bf3113172485cd6ffc52ccc6a5e8512b490',
+                'aws_access_key': 'AKIARBAGHTGOZC7544GN',
+                'aws_secret_key': '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',
+                'role_arn': 'arn:aws:iam::070880041373:role/Amazon_SP_API_ROLE'
+            }
+
+            single_info = df.query("refresh_token==@refresh_token")
+            region_circle = single_info['region'].values[0]
+            seller_id = single_info['selling_partner_id'].values[0]
+            account_name = single_info['account_name'].values[0]
+            if region_circle == 'NA':
+                pass
+                for marketplace in [Marketplaces.US, Marketplaces.BR, Marketplaces.CA, Marketplaces.MX]:
+                    sp_api = SpApiRequest(aws_credentials, marketplace)
+                    try:
+                        sp_api.GET_FLAT_FILE_OPEN_LISTINGS_DATA(auth_conn, seller_id)
+                    except Exception as e:
+                        print(e)
+            elif region_circle == 'EU':
+                pass
+                for marketplace in [Marketplaces.DE,Marketplaces.AE, Marketplaces.BE,  Marketplaces.PL,
+                                    Marketplaces.EG,Marketplaces.ES,  Marketplaces.GB, Marketplaces.IN, Marketplaces.IT,
+                                    Marketplaces.NL, Marketplaces.SA, Marketplaces.SE, Marketplaces.TR,Marketplaces.UK,Marketplaces.FR,
+
+                                    ]:
+                    sp_api = SpApiRequest(aws_credentials, marketplace)
+                    try:
+                        sp_api.GET_FLAT_FILE_OPEN_LISTINGS_DATA(auth_conn, seller_id)
+                    except Exception as e:
+                        print(e)
+
+            else:
+                marketplace = eval(f'Marketplaces.{region_circle}')
+                sp_api = SpApiRequest(aws_credentials, marketplace)
+                sp_api.GET_FLAT_FILE_OPEN_LISTINGS_DATA(auth_conn, seller_id)
+
+    def timeDeal(self, orgTime):
+        orgTime = parse(orgTime)
+        timezone = pytz.timezone("UTC")
+        shopTime = orgTime.astimezone(timezone)
+        shopTime_datetime = datetime(shopTime.year, shopTime.month, shopTime.day, shopTime.hour, shopTime.minute,
+                                     shopTime.second)
+        return shopTime_datetime
+
+    def GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL(self,seller_id):
+        # timezone_ = pytz.timezone(self.timezone)
+        shopReportday = (datetime.now() + timedelta(days=-2)).strftime("%Y-%m-%d")
         # print(shopReportday)
         para = {"reportType":ReportType.GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL,"dataStartTime":shopReportday,"dataEndTime":shopReportday,"reportOptions":{"ShowSalesChannel":"true"}}
         reportid = self.create_report(**para) #{"ShowSalesChannel":"true"}
         decom_df = self.decompression(reportid)
-        decom_df[["purchase-date","last-updated-date"]] = decom_df[["purchase-date","last-updated-date"]].applymap(lambda x: self.timeDeal(x) if pd.isna(x)==False or x != None else x)
 
         decom_df[decom_df.select_dtypes(float).columns] = decom_df[decom_df.select_dtypes(float).columns].fillna(0.0)
         decom_df[decom_df.select_dtypes(int).columns] = decom_df[decom_df.select_dtypes(int).columns].fillna(0)
@@ -269,9 +488,13 @@ class SpApiRequest:
         if "purchase-order-number" in decom_df.columns:
             decom_df['purchase-order-number'] = decom_df['purchase-order-number'].astype("string")
         decom_df.fillna('',inplace=True)
+        # decom_df.to_csv('order.csv')
         decom_df["ReportDate"] = parse(shopReportday)
-        decom_df['timezone'] = self.timezone
-        decom_df['profileid'] = str(self.profileid)
+        decom_df['timezone'] =  decom_df[["purchase-date"]].map(lambda x: parse(x).tzname()).fillna(method='bfill')
+        decom_df[["purchase-date", "last-updated-date"]] = decom_df[["purchase-date", "last-updated-date"]].applymap(
+            lambda x: self.timeDeal(x) if pd.isna(x) == False or x != None else x)
+
+        decom_df['seller_id'] = seller_id
         list_df = decom_df.to_numpy().tolist()
         print(list_df[0])
         # tuple_data = [tuple(i) for i in list_df]
@@ -292,251 +515,84 @@ class SpApiRequest:
             conn.rollback()
             print(e)
 
-    @throttle_retry()
-    @load_all_pages()
-    def load_all_orders(self,**kwargs):
-        """
-        a generator function to return all pages, obtained by NextToken
-        """
-        return Orders(credentials=self.credentials, marketplace=Marketplaces.US).get_orders(**kwargs)
-
-    @throttle_retry()
-    @load_all_pages()
-    def load_order_items(self,**kwargs):
-        return Orders(credentials=self.credentials).get_order_items(**kwargs)
-
-    def order_instant(self, sleep_time=0.8):
-        insert_list_orderBasic = []
-        insert_list_orderDetail = []
-        orderId_list = []
-        row = 0
-        dt_ = datetime.utcnow()
-        LastUpdatedAfter = (datetime(dt_.year, dt_.month, dt_.day, dt_.hour, 0, 0, 0) - timedelta(hours=1)).isoformat()
-        LastUpdatedBefore = (datetime(dt_.year, dt_.month, dt_.day, dt_.hour, 59, 59, 59) - timedelta(hours=1)).isoformat()
-        print(LastUpdatedAfter, '-', LastUpdatedBefore)
-        for page in self.load_all_orders(LastUpdatedAfter=LastUpdatedAfter, LastUpdatedBefore=LastUpdatedBefore):  #
-            for order in page.payload.get('Orders'):
-                # print(order)
-                AmazonOrderId = str(order.get('AmazonOrderId'))
-                orderId_list.append(AmazonOrderId)
-                try:
-                    items_ = self.load_order_items(order_id=str(order.get('AmazonOrderId')))
-                except:
-                    try:
-                        time.sleep(3)
-                        items_ = self.load_order_items(order_id=str(order.get('AmazonOrderId')))
-                    except:
-                        time.sleep(5)
-                        items_ = self.load_order_items(order_id=str(order.get('AmazonOrderId')))
-                # print(next(items_).payload)
-                items_detail = next(items_).payload["OrderItems"][0]
-                TaxCollection_Model, TaxCollection_ResponsibleParty = (None, None) if items_detail.get(
-                    "TaxCollection") == None else (items_detail.get("TaxCollection").get("Model"),
-                                                   items_detail.get("TaxCollection").get("ResponsibleParty"))
-                ProductInfo_NumberOfItems = None if items_detail.get("ProductInfo") == None else int(
-                    items_detail.get("ProductInfo").get("NumberOfItems"))
-                BuyerInfo = None if items_detail.get("BuyerInfo") == None else str(items_detail.get("BuyerInfo"))
-                CurrencyCode = None if items_detail.get("ItemPrice") == None else items_detail.get("ItemPrice").get(
-                    "CurrencyCode")
-                ItemTax_Amount = None if items_detail.get("ItemTax") == None else float(
-                    items_detail.get("ItemTax").get("Amount"))
-                QuantityShipped = None if items_detail.get("QuantityShipped") == None else int(
-                    items_detail.get("QuantityShipped"))
-                ItemPrice_Amount = None if items_detail.get("ItemPrice") == None else float(
-                    items_detail.get("ItemPrice").get("Amount"))
-                ASIN = None if items_detail.get("ASIN") == None else items_detail.get("ASIN")
-                SellerSKU = None if items_detail.get("SellerSKU") == None else items_detail.get("SellerSKU")
-                Title = None if items_detail.get("Title") == None else items_detail.get("Title")
-                ShippingTax_Amount = None if items_detail.get("ShippingTax") == None else float(
-                    items_detail.get("ShippingTax").get("Amount"))
-                IsGift = None if items_detail.get("IsGift") == None else (
-                    False if items_detail.get("IsGift") == 'false' else True)
-                PriceDesignation = None if items_detail.get("PriceDesignation") == None else items_detail.get(
-                    "PriceDesignation")
-                ShippingPrice_Amount = None if items_detail.get("ShippingPrice") == None else float(
-                    items_detail.get("ShippingPrice").get("Amount"))
-                ShippingDiscount_Amount = None if items_detail.get("ShippingDiscount") == None else float(
-                    items_detail.get("ShippingDiscount").get("Amount"))
-                ShippingDiscountTax_Amount = None if items_detail.get("ShippingDiscountTax") == None else float(
-                    items_detail.get("ShippingDiscountTax").get("Amount"))
-                IsTransparency = None if items_detail.get("IsTransparency") == None else items_detail.get(
-                    "IsTransparency")
-                QuantityOrdered = None if items_detail.get("QuantityOrdered") == None else int(
-                    items_detail.get("QuantityOrdered"))
-                PromotionDiscountTax_Amount = None if items_detail.get("PromotionDiscountTax") == None else float(
-                    items_detail.get("PromotionDiscountTax").get("Amount"))
-                PromotionDiscount_Amount = None if items_detail.get("PromotionDiscount") == None else float(
-                    items_detail.get("PromotionDiscount").get("Amount"))
-                OrderItemId = None if items_detail.get("OrderItemId") == None else str(items_detail.get("OrderItemId"))
-                temp_inserttime = datetime.now(tz=pytz.timezone(self.timezone)) + timedelta(hours=1)
-                Inserthour_time = datetime(temp_inserttime.year, temp_inserttime.month, temp_inserttime.day,temp_inserttime.hour, 0, 0)
-                insert_list_orderDetail.append(
-                    [AmazonOrderId, TaxCollection_Model, TaxCollection_ResponsibleParty, ProductInfo_NumberOfItems,
-                     BuyerInfo,
-                     CurrencyCode, ItemTax_Amount, QuantityShipped, ItemPrice_Amount, ASIN, SellerSKU, Title,
-                     ShippingTax_Amount, IsGift, PriceDesignation, ShippingPrice_Amount,
-                     ShippingDiscount_Amount, ShippingDiscountTax_Amount, IsTransparency, QuantityOrdered,
-                     PromotionDiscountTax_Amount, PromotionDiscount_Amount, OrderItemId, Inserthour_time,self.timezone])
-
-                BuyerInfo_BuyerEmail = None if order.get('BuyerInfo') == None else order.get('BuyerInfo').get(
-                    'BuyerEmail')
-                AmazonOrderId = AmazonOrderId  # order.get('AmazonOrderId') == None else order.get('AmazonOrderId')
-
-                EarliestDeliveryDate = None if order.get('EarliestDeliveryDate') == None else self.timeDeal(order.get('EarliestDeliveryDate'))
-
-                EarliestShipDate = None if order.get('EarliestShipDate') == None else self.timeDeal(order.get('EarliestShipDate'))
-
-                SalesChannel = None if order.get('SalesChannel') == None else order.get('SalesChannel')
-                AutomatedShippingSettings_HasAutomatedShippingSettings = None if order.get('AutomatedShippingSettings') == None else order.get('AutomatedShippingSettings').get('HasAutomatedShippingSettings')
-                OrderStatus = None if order.get('OrderStatus') == None else order.get('OrderStatus')
-                NumberOfItemsShipped = None if order.get('NumberOfItemsShipped') == None else order.get('NumberOfItemsShipped')
-                OrderType = None if order.get('OrderType') == None else order.get('OrderType')
-                IsPremiumOrder = None if order.get('IsPremiumOrder') == None else order.get('IsPremiumOrder')
-                IsPrime = None if order.get('IsPrime') == None else order.get('IsPrime')
-                FulfillmentChannel = None if order.get('FulfillmentChannel') == None else order.get('FulfillmentChannel')
-                NumberOfItemsUnshipped = None if order.get('NumberOfItemsUnshipped') == None else order.get('NumberOfItemsUnshipped')
-                HasRegulatedItems = None if order.get('HasRegulatedItems') == None else order.get('HasRegulatedItems')
-                IsReplacementOrder = None if order.get('IsReplacementOrder') == None else eval(order.get('IsReplacementOrder').capitalize())
-                IsSoldByAB = None if order.get('IsSoldByAB') == None else order.get('IsSoldByAB')
-                LatestShipDate = None if order.get('LatestShipDate') == None else self.timeDeal(order.get('LatestShipDate'))
-                ShipServiceLevel = None if order.get('ShipServiceLevel') == None else order.get('ShipServiceLevel')
-                DefaultShipFromLocationAddress_StateOrRegion = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('StateOrRegion')
-                DefaultShipFromLocationAddress_AddressLine1 = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('AddressLine1')
-                DefaultShipFromLocationAddress_Phone = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('Phone')
-                DefaultShipFromLocationAddress_PostalCode = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('PostalCode')
-                DefaultShipFromLocationAddress_City = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('City')
-                DefaultShipFromLocationAddress_CountryCode = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('CountryCode')
-                DefaultShipFromLocationAddress_Name = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('Name')
-                IsISPU = None if order.get('IsISPU') == None else order.get('IsISPU')
-                MarketplaceId = None if order.get('MarketplaceId') == None else order.get('MarketplaceId')
-                LatestDeliveryDate = None if order.get('LatestDeliveryDate') == None else self.timeDeal(order.get('LatestDeliveryDate'))
-                PurchaseDate = None if order.get('PurchaseDate') == None else self.timeDeal(order.get('PurchaseDate'))
-                ShippingAddress_StateOrRegion = None if order.get('ShippingAddress') == None else order.get('ShippingAddress').get('StateOrRegion')
-                ShippingAddress_PostalCode = None if order.get('ShippingAddress') == None else order.get('ShippingAddress').get('PostalCode')
-                ShippingAddress_City = None if order.get('ShippingAddress') == None else order.get('ShippingAddress').get('City')
-                ShippingAddress_CountryCode = None if order.get('ShippingAddress') == None else order.get('ShippingAddress').get('CountryCode')
-                IsAccessPointOrder = None if order.get('IsAccessPointOrder') == None else order.get('IsAccessPointOrder')
-                PaymentMethod = None if order.get('PaymentMethod') == None else order.get('PaymentMethod')
-                IsBusinessOrder = None if order.get('IsBusinessOrder') == None else order.get('IsBusinessOrder')
-                OrderTotal_CurrencyCode = None if order.get('OrderTotal') == None else order.get('OrderTotal').get('CurrencyCode')
-                OrderTotal_Amount = None if order.get('OrderTotal') == None else float(order.get('OrderTotal').get('Amount'))
-                PaymentMethodDetails = None if order.get('PaymentMethodDetails') == None else order.get('PaymentMethodDetails')[0]
-                IsGlobalExpressEnabled = None if order.get('IsGlobalExpressEnabled') == None else order.get('IsGlobalExpressEnabled')
-                LastUpdateDate = None if order.get('LastUpdateDate') == None else self.timeDeal(order.get('LastUpdateDate'))
-                ShipmentServiceLevelCategory = None if order.get('ShipmentServiceLevelCategory') == None else order.get('ShipmentServiceLevelCategory')
-                SellerOrderId = None if order.get('SellerOrderId') == None else order.get('SellerOrderId')
-                temp_inserttime =  datetime.now(tz=pytz.timezone(self.timezone))+timedelta(hours=1)
-                Inserthour_time = datetime(temp_inserttime.year,temp_inserttime.month,temp_inserttime.day,temp_inserttime.hour,0,0)
-
-                insert_list_orderBasic.append(
-                    [BuyerInfo_BuyerEmail, AmazonOrderId, EarliestDeliveryDate, EarliestShipDate, SalesChannel,
-                     AutomatedShippingSettings_HasAutomatedShippingSettings, OrderStatus,
-                     NumberOfItemsShipped, OrderType, IsPremiumOrder, IsPrime, FulfillmentChannel,
-                     NumberOfItemsUnshipped, HasRegulatedItems, IsReplacementOrder, IsSoldByAB, LatestShipDate,
-                     ShipServiceLevel, DefaultShipFromLocationAddress_StateOrRegion,
-                     DefaultShipFromLocationAddress_AddressLine1, DefaultShipFromLocationAddress_Phone,
-                     DefaultShipFromLocationAddress_PostalCode,
-                     DefaultShipFromLocationAddress_City, DefaultShipFromLocationAddress_CountryCode,
-                     DefaultShipFromLocationAddress_Name, IsISPU, MarketplaceId, LatestDeliveryDate, PurchaseDate,
-                     ShippingAddress_StateOrRegion, ShippingAddress_PostalCode, ShippingAddress_City,
-                     ShippingAddress_CountryCode, IsAccessPointOrder, PaymentMethod, IsBusinessOrder,
-                     OrderTotal_CurrencyCode,
-                     OrderTotal_Amount, PaymentMethodDetails, IsGlobalExpressEnabled, LastUpdateDate,
-                     ShipmentServiceLevelCategory, SellerOrderId, Inserthour_time,self.timezone])
-
-                time.sleep(sleep_time)
-                row += 1
-                if row % 10 == 0:
-                    print(f"receiving...received {row} rows")
-        print(insert_list_orderBasic)
-        print(insert_list_orderDetail)
+
 if __name__ == '__main__':
-    aws_credentials = {
-        'refresh_token': 'Atzr|IwEBIMxC7d17ZYBTcNe-zfnbk-TEC-40uIlSRmGAH_sfNozcA7RdSt4iBdUorC2GC_uoUmIY4oGhHGT621el1my0YaABEvuGn4eIe1EFxrHYLM1fljnQxHQjgGKwTb48cMdHNxOiMd8_CbvLYL_NT3E_zTPKCSQjqa8zxTkddBjL-5stlTRzhVHmB2Ox5-6XgvpSnoBtlZqkI96rYmKi63f0NTQ2e9IV3cNqXli8X9_DYGgZlpl60qE56A-ZNy4otv4myR5kqY2bTll0c-ynLtqI5ukDwPwO369b_Ie4kniRd7Or-Ip6jPyTIdfghLV4AVrxbvE',
-        'lwa_app_id': 'amzn1.application-oa2-client.1f9d3d4747e14b22b4b598e54e6b922e',  # 卖家中心里面开发者资料LWA凭证
-        'lwa_client_secret': 'amzn1.oa2-cs.v1.3af0f5649f5b8e151cd5bd25c10f2bf3113172485cd6ffc52ccc6a5e8512b490',
-        'aws_access_key': 'AKIARBAGHTGOZC7544GN',
-        'aws_secret_key': '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',
-        'role_arn': 'arn:aws:iam::070880041373:role/Amazon_SP_API_ROLE'
-    }
-    sp_ = SpApiRequest(aws_credentials,Marketplaces.CA,'3006125408623189')
-    # print(sp_.shopInfo)
-    print(str(Marketplaces.CA)[-2:])
-    # for i in Marketplaces:
-    #     print(i.marketplace_id)
-    sp_.GET_FLAT_FILE_OPEN_LISTINGS_DATA()
-    """
-    create database amz_sp_api;
-    """
-    """
-            create table amz_sp_api.productInfo
-            (
-            `item-name`	VARCHAR(300),
-            `item-description`	VARCHAR(1000),
-            `listing-id`	VARCHAR(50),
-            `seller-sku`	VARCHAR(50),
-            `price`	FLOAT,
-            `quantity`	INT,
-            `open-date`	VARCHAR(70),
-            `image-url`	VARCHAR(300),
-            `item-is-marketplace`	VARCHAR(50),
-            `product-id-type`	INT,
-            `item-note`	VARCHAR(300),
-            `item-condition`	INT,
-            `asin1`	VARCHAR(50),
-            `asin2`	VARCHAR(50),
-            `asin3`	VARCHAR(50),
-            `will-ship-internationally`	VARCHAR(50),
-            `expedited-shipping`	VARCHAR(50),
-            `product-id`	VARCHAR(50),
-            `bid-for-featured-placement`	FLOAT,
-            `add-delete`	VARCHAR(50),
-            `pending-quantity`	INT,
-            `fulfillment-channel`	VARCHAR(50),
-            `merchant-shipping-group`	VARCHAR(50),
-            `status`	VARCHAR(50),
-            `mainImageUrl`	VARCHAR(300),
-            `opendate_date`	Date,
-            `updateTime`	Date,
-            `timezone`	VARCHAR(30)
-            )
-    """
-    """
-    create table amz_sp_api.orderReport
-    (`amazon-order-id` VARCHAR(40),
-    `merchant-order-id` VARCHAR(40),
-    `purchase-date` DATETIME,
-    `last-updated-date` DATETIME,
-    `order-status` VARCHAR(40),
-    `fulfillment-channel` VARCHAR(40),
-    `sales-channel` VARCHAR(40),
-    `order-channel` VARCHAR(40),
-    `ship-service-level` VARCHAR(40),
-    `product-name` VARCHAR(250),
-    `sku` VARCHAR(50),
-    `asin` VARCHAR(40),
-    `item-status` VARCHAR(40),
-    `quantity` INT,
-    `currency` VARCHAR(40),
-    `item-price` FLOAT,
-    `item-tax` FLOAT,
-    `shipping-price` FLOAT,
-    `shipping-tax` FLOAT,
-    `gift-wrap-price` FLOAT,
-    `gift-wrap-tax` FLOAT,
-    `item-promotion-discount` FLOAT,
-    `ship-promotion-discount` FLOAT,
-    `ship-city` VARCHAR(40),
-    `ship-state` VARCHAR(40),
-    `ship-postal-code` VARCHAR(40),
-    `ship-country` VARCHAR(40),
-    `promotion-ids` VARCHAR(50),
-    `cpf` VARCHAR(40),
-    `is-business-order` BOOL,
-    `purchase-order-number` VARCHAR(50),
-    `price-designation` VARCHAR(40),
-    `signature-confirmation-recommended` BOOL,
-    `ReportDate` DATE not null,
-    `timezone` VARCHAR(20) not null
-    );
-    """
+    SpApiRequest.get_allShops()
+
+
+
+"""
+create database amz_sp_api;
+"""
+"""
+        create table amz_sp_api.productInfo
+        (
+        `item-name`	VARCHAR(300),
+        `item-description`	VARCHAR(1000),
+        `listing-id`	VARCHAR(50),
+        `seller-sku`	VARCHAR(50),
+        `price`	FLOAT,
+        `quantity`	INT,
+        `open-date`	VARCHAR(70),
+        `image-url`	VARCHAR(300),
+        `item-is-marketplace`	VARCHAR(50),
+        `product-id-type`	INT,
+        `item-note`	VARCHAR(300),
+        `item-condition`	INT,
+        `asin1`	VARCHAR(50),
+        `asin2`	VARCHAR(50),
+        `asin3`	VARCHAR(50),
+        `will-ship-internationally`	VARCHAR(50),
+        `expedited-shipping`	VARCHAR(50),
+        `product-id`	VARCHAR(50),
+        `bid-for-featured-placement`	FLOAT,
+        `add-delete`	VARCHAR(50),
+        `pending-quantity`	INT,
+        `fulfillment-channel`	VARCHAR(50),
+        `merchant-shipping-group`	VARCHAR(50),
+        `status`	VARCHAR(50),
+        `mainImageUrl`	VARCHAR(300),
+        `opendate_date`	Date,
+        `updateTime`	Date,
+        `timezone`	VARCHAR(30)
+        )
+"""
+"""
+create table amz_sp_api.orderReport
+(`amazon-order-id` VARCHAR(40),
+`merchant-order-id` VARCHAR(40),
+`purchase-date` DATETIME,
+`last-updated-date` DATETIME,
+`order-status` VARCHAR(40),
+`fulfillment-channel` VARCHAR(40),
+`sales-channel` VARCHAR(40),
+`order-channel` VARCHAR(40),
+`ship-service-level` VARCHAR(40),
+`product-name` VARCHAR(250),
+`sku` VARCHAR(50),
+`asin` VARCHAR(40),
+`item-status` VARCHAR(40),
+`quantity` INT,
+`currency` VARCHAR(40),
+`item-price` FLOAT,
+`item-tax` FLOAT,
+`shipping-price` FLOAT,
+`shipping-tax` FLOAT,
+`gift-wrap-price` FLOAT,
+`gift-wrap-tax` FLOAT,
+`item-promotion-discount` FLOAT,
+`ship-promotion-discount` FLOAT,
+`ship-city` VARCHAR(40),
+`ship-state` VARCHAR(40),
+`ship-postal-code` VARCHAR(40),
+`ship-country` VARCHAR(40),
+`promotion-ids` VARCHAR(50),
+`cpf` VARCHAR(40),
+`is-business-order` BOOL,
+`purchase-order-number` VARCHAR(50),
+`price-designation` VARCHAR(40),
+`signature-confirmation-recommended` BOOL,
+`ReportDate` DATE not null,
+`timezone` VARCHAR(20) not null
+);
+"""