Merge branch 'yifan' of ASJ_ADS/sync_amz_data into master

sync_amz_data/public/sp_api_client.py

@@ -29,29 +29,24 @@ class SpApiRequest:
     def __init__(self, credentials,marketplace):
         self.credentials = credentials
         self.marketplace = marketplace
-        # self.shopInfo = shop_infos('3006125408623189')
-        # self.timezone = self.shopInfo['time_zone']
-        # self.profileid = '3006125408623189'
 
-    #
     @classmethod
     @retry(tries=3, delay=5, backoff=2, )
-    def mysql_connect_auth(cls): # AUTH
+    def Token_auth(cls): # AUTH - mysql_connect_auth
         conn = pymysql.connect(**MYSQL_AUTH_CONF)
-
         return conn
 
 
     @classmethod
     @retry(tries=3, delay=5, backoff=2, )
-    def mysql_connect_auth_lst(cls): # DATA
+    def Data_auth(cls): # DATA - mysql_connect_auth_lst
         conn = pymysql.connect(**MYSQL_DATA_CONF)
         return conn
 
 
     @classmethod
     @retry(tries=3, delay=5, backoff=2, )
-    def mysql_connect(cls): # local database
+    def LocalHost_Auth(cls): # local database-
         conn = pymysql.connect(user="huangyifan",
                                password="123456",
                                host="127.0.0.1",
@@ -62,8 +57,8 @@ class SpApiRequest:
 
     @staticmethod
     @retry(tries=3, delay=5, backoff=2, )
-    def auth_info(): # get Auth-data from all of shops
-        auth_conn = SpApiRequest.mysql_connect_auth()
+    def auth_info(): # get Auth-data from all of shops - 此连接获取refreshtoken供账户授权使用
+        auth_conn = SpApiRequest.Token_auth()
         cursor = auth_conn.cursor()
         cursor.execute("select * from amazon_sp_report.amazon_sp_auth_info;")
         columns_name = [i[0] for i in cursor.description]
@@ -74,13 +69,13 @@ class SpApiRequest:
 
     @classmethod
     @retry(tries=3, delay=5, backoff=2, )
-    def get_refreshtoken(cls): # accroding to differnt shop get diffrent refreshtoken
+    def get_refreshtoken(cls): # accroding to differnt shop get diffrent refreshtoken - 获取授权后拿到refreshtoken
         df = cls.auth_info()
         refreshtoken_list = (df['refresh_token'].to_numpy().tolist())
         return refreshtoken_list
 
     @classmethod
-    def get_catelog(cls,account_name,country=Marketplaces.US,asin=None): # desprecated
+    def get_catelog(cls,account_name,country=Marketplaces.US,asin=None): # active - 可以使用但未进行使用
         if country in [Marketplaces.US, Marketplaces.BR, Marketplaces.CA,Marketplaces.MX]:
             region = 'NA'
         elif country in [Marketplaces.DE,Marketplaces.AE, Marketplaces.BE,  Marketplaces.PL,
@@ -104,14 +99,20 @@ class SpApiRequest:
                 'aws_secret_key': 'OSbkKKjShvDoWGBwRORSUqDryBtKWs8AckzwNMzR',
                 'role_arn': 'arn:aws:iam::070880041373:role/Amazon_SP_API_ROLE'
             }
-        cate_item = CatalogItems(credentials=cred, marketplace=country)
-        images_info = cate_item.get_catalog_item(asin=asin,**{"includedData":['images']})
-        images = images_info.images[0].get('images')[0]['link']
-        title_info = cate_item.get_catalog_item(asin=asin)
-        title = title_info.payload['summaries'][0]['itemName']
-        return {'images':images,'title':title}
-
-    def create_report(self,**kwargs): # Main-CreateReport-Function
+        # cate_item = CatalogItems(credentials=cred, marketplace=country)
+        # images_info = cate_item.get_catalog_item(asin=asin,**{"includedData":['images']})
+        # print(images_info)
+        # images = images_info.images[0].get('images')[0]['link']
+        # title_info = cate_item.get_catalog_item(asin=asin)
+        # print(title_info)
+        # title = title_info.payload['summaries'][0]['itemName']
+        cate_item = CatalogItems(credentials=cred, marketplace=country,version='2022-04-01')
+        test_bundle = cate_item.get_catalog_item(asin=asin,**{"includedData":['salesRanks']})
+        print(test_bundle)
+        # return {'images':images,'title':title}
+
+    @retry(tries=3, delay=5, backoff=2,)
+    def create_report(self,**kwargs): # Main-CreateReport-Function - 创建报告请求
         reportType = kwargs['reportType']
         reportOptions =kwargs.get("reportOptions")
 
@@ -127,7 +128,8 @@ class SpApiRequest:
         # print(reportId)
         return reportId
 
-    def decompression(self,reportId): # After-CreateReportFunc-simpleDeal
+    @retry(tries=2, delay=3, backoff=2,)
+    def decompression(self,reportId): # After-CreateReportFunc-simpleDeal - 根据获取到的报告id进行解压获取
         report = Reports(credentials=self.credentials, marketplace=self.marketplace)
         while True:
             reportId_info = report.get_report(reportId=reportId)
@@ -159,129 +161,128 @@ class SpApiRequest:
             print("please wait...")
 
 
-    def GET_MERCHANT_LISTINGS_ALL_DATA(self,limit=None): # Not be used
-        start = time.time()
-        para = {"reportType":ReportType.GET_MERCHANT_LISTINGS_ALL_DATA}
-        reportid = self.create_report(**para)
-        decom_df = self.decompression(reportid)
-        print("连接数据库")
-        conn = self.mysql_connect()
-        print("连接成功")
-        cursor = conn.cursor()
-        timezone = "UTC" #pytz.timezone(self.timezone)
-        bondary_date = (datetime.now()).strftime("%Y-%m-%d") #+ timedelta(days=-28)
-        cursor.execute(f"""select * from amz_sp_api.productInfo where (mainImageUrl is not null and mainImageUrl not in ('', ' ')) and 
-                        (`seller-sku` not in ('',' ') and `seller-sku` is not null) and 
-                        `updateTime`>='{bondary_date}'""") #`seller-sku`,`updateTime`,`mainImageUrl`
-        col = [i[0] for i in cursor.description]
-        query_rel = cursor.fetchall()
-
-        if len(query_rel)!=0:
-            print(query_rel[0])
-            df = pd.DataFrame(query_rel,columns=col)
-            listingid = df['listing-id'].to_numpy().tolist()
-            decom_df = decom_df.query("`listing-id` not in @listingid")
-            print("数据条数: ",len(decom_df))
-            # print(f"delete * from amz_sp_api.productInfo where `listing-id` not in {tuple(listingid)}")
-
-            # conn.commit()
-
-        if len(decom_df)==0:
-            return "Done"
-
-        if limit != None:
-            decom_df = decom_df.iloc[:limit,:]
-        print("getting mainImageInfo...")
-        rowcount = 0
-        while rowcount < len(decom_df):
-            df_insert = decom_df.copy()
-            df_insert = df_insert.iloc[rowcount:rowcount + 200, :]
-
-            df_insert = self.data_deal(df_insert)
-            list_df = df_insert.to_numpy().tolist()
-
-            # print(list(conn.query("select * from amz_sp_api.orderReport")))
-            sql = f"""
-                        insert into amz_sp_api.productInfo
-                        values (%s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s,%s)
-                    """ #ok
-            # print(sql)
-            conn = self.mysql_connect()
-            cursor = conn.cursor()
-            try:
-                conn.begin()
-                cursor.executemany(sql, list_df)
-                conn.commit()
-                print("插入中...")
-                insert_listingid = df_insert['listing-id'].to_numpy().tolist()
-                cursor.execute(f"delete from amz_sp_api.productInfo where `listing-id` not in {tuple(insert_listingid)} and `updateTime`<'{bondary_date}'")
-                conn.commit()
-                rowcount += 200
-            except Exception as e:
-                conn.rollback()
-                print(e)
-                try:
-                    conn = self.mysql_connect()
-                    cursor = conn.cursor()
-                    conn.begin()
-                    cursor.executemany(sql, list_df)
-                    conn.commit()
-                    insert_listingid = df_insert['listing-id'].to_numpy().tolist()
-                    cursor.execute(f"delete from amz_sp_api.productInfo where `listing-id` not in {tuple(insert_listingid)} and `updateTime`<'{bondary_date}'")
-                    conn.commit()
-                except Exception as e:
-                    conn.rollback()
-                    print(e)
-                    break
-                # break
-        conn.close()
-        print("全部完成")
-        end =time.time()
-        print("duration:",end-start)
-        return decom_df
-
-    def data_deal(self, decom_df, seller_id): # desprecated
-        decom_df['mainImageUrl'] = decom_df['seller-sku'].map(lambda x: self.get_mainImage_url(x))
-        url_columns = [i for i in decom_df.columns if "url" in i.lower()]
-        if len(url_columns) > 0:
-            decom_df[url_columns] = decom_df[url_columns].astype("string")
-        asin_columns = [i for i in decom_df.columns if 'asin' in i.lower()]
-        if len(asin_columns) > 0:
-            decom_df[asin_columns] = decom_df[asin_columns].astype("string")
-        if 'pending-quantity' in decom_df.columns:
-            decom_df['pending-quantity'] = decom_df['pending-quantity'].map(
-                lambda x: 0 if pd.isna(x) or np.isinf(x) else x).astype("int32")
-        deletecolumns = [i for i in decom_df.columns if 'zshop' in i.lower()]
-        decom_df.drop(columns=deletecolumns, inplace=True)
-        if 'quantity' in decom_df.columns:
-            decom_df['quantity'] = decom_df['quantity'].map(lambda x: 0 if pd.isna(x) or np.isinf(x) else x).astype(
-                "int32")
-        decom_df['opendate_date'] = decom_df['open-date'].map(lambda x: self.datetime_deal(x))
-        if 'add-delete' in decom_df.columns:
-            decom_df['add-delete'] = decom_df['add-delete'].astype('string', errors='ignore')
-        if 'will-ship-internationally' in decom_df.columns:
-            decom_df['will-ship-internationally'] = decom_df['will-ship-internationally'].astype('string',
-                                                                                                 errors='ignore')
-        if 'expedited-shipping' in decom_df.columns:
-            decom_df['expedited-shipping'] = decom_df['expedited-shipping'].astype('string', errors='ignore')
-        decom_df['updateTime'] = datetime.now()
-        decom_df['timezone'] = "UTC"
-        decom_df['seller_id'] = seller_id
-        #
-        decom_df['item-description'] = decom_df['item-description'].str.slice(0, 500)
-        decom_df[decom_df.select_dtypes(float).columns] = decom_df[decom_df.select_dtypes(float).columns].fillna(0.0)
-        decom_df[decom_df.select_dtypes(int).columns] = decom_df[decom_df.select_dtypes(int).columns].fillna(0)
-        decom_df[decom_df.select_dtypes(datetime).columns] = decom_df[decom_df.select_dtypes(datetime).columns].astype(
-            'string')
-        decom_df.fillna('', inplace=True)
-        # print(decom_df.info())
-        return decom_df
-
-    def GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA(self,refresh_token,conn=None,seller_id=None,days=-1,**kwargs):
+    # def GET_MERCHANT_LISTINGS_ALL_DATA(self,limit=None): # Not be used
+    #     start = time.time()
+    #     para = {"reportType":ReportType.GET_MERCHANT_LISTINGS_ALL_DATA}
+    #     reportid = self.create_report(**para)
+    #     decom_df = self.decompression(reportid)
+    #     print("连接数据库")
+    #     conn = self.LocalHost_Auth()
+    #     print("连接成功")
+    #     cursor = conn.cursor()
+    #     timezone = "UTC" #pytz.timezone(self.timezone)
+    #     bondary_date = (datetime.now()).strftime("%Y-%m-%d") #+ timedelta(days=-28)
+    #     cursor.execute(f"""select * from amz_sp_api.productInfo where (mainImageUrl is not null and mainImageUrl not in ('', ' ')) and
+    #                     (`seller-sku` not in ('',' ') and `seller-sku` is not null) and
+    #                     `updateTime`>='{bondary_date}'""") #`seller-sku`,`updateTime`,`mainImageUrl`
+    #     col = [i[0] for i in cursor.description]
+    #     query_rel = cursor.fetchall()
+    #
+    #     if len(query_rel)!=0:
+    #         print(query_rel[0])
+    #         df = pd.DataFrame(query_rel,columns=col)
+    #         listingid = df['listing-id'].to_numpy().tolist()
+    #         decom_df = decom_df.query("`listing-id` not in @listingid")
+    #         print("数据条数: ",len(decom_df))
+    #         # print(f"delete * from amz_sp_api.productInfo where `listing-id` not in {tuple(listingid)}")
+    #
+    #         # conn.commit()
+    #
+    #     if len(decom_df)==0:
+    #         return "Done"
+    #
+    #     if limit != None:
+    #         decom_df = decom_df.iloc[:limit,:]
+    #     print("getting mainImageInfo...")
+    #     rowcount = 0
+    #     while rowcount < len(decom_df):
+    #         df_insert = decom_df.copy()
+    #         df_insert = df_insert.iloc[rowcount:rowcount + 200, :]
+    #
+    #         df_insert = self.data_deal(df_insert)
+    #         list_df = df_insert.to_numpy().tolist()
+    #
+    #         # print(list(conn.query("select * from amz_sp_api.orderReport")))
+    #         sql = f"""
+    #                     insert into amz_sp_api.productInfo
+    #                     values (%s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s,%s)
+    #                 """ #ok
+    #         # print(sql)
+    #         conn = self.LocalHost_Auth()
+    #         cursor = conn.cursor()
+    #         try:
+    #             conn.begin()
+    #             cursor.executemany(sql, list_df)
+    #             conn.commit()
+    #             print("插入中...")
+    #             insert_listingid = df_insert['listing-id'].to_numpy().tolist()
+    #             cursor.execute(f"delete from amz_sp_api.productInfo where `listing-id` not in {tuple(insert_listingid)} and `updateTime`<'{bondary_date}'")
+    #             conn.commit()
+    #             rowcount += 200
+    #         except Exception as e:
+    #             conn.rollback()
+    #             print(e)
+    #             try:
+    #                 conn = self.LocalHost_Auth()
+    #                 cursor = conn.cursor()
+    #                 conn.begin()
+    #                 cursor.executemany(sql, list_df)
+    #                 conn.commit()
+    #                 insert_listingid = df_insert['listing-id'].to_numpy().tolist()
+    #                 cursor.execute(f"delete from amz_sp_api.productInfo where `listing-id` not in {tuple(insert_listingid)} and `updateTime`<'{bondary_date}'")
+    #                 conn.commit()
+    #             except Exception as e:
+    #                 conn.rollback()
+    #                 print(e)
+    #                 break
+    #             # break
+    #     conn.close()
+    #     print("全部完成")
+    #     end =time.time()
+    #     print("duration:",end-start)
+    #     return decom_df
+
+    # def data_deal(self, decom_df, seller_id): # desprecated
+    #     decom_df['mainImageUrl'] = decom_df['seller-sku'].map(lambda x: self.get_mainImage_url(x))
+    #     url_columns = [i for i in decom_df.columns if "url" in i.lower()]
+    #     if len(url_columns) > 0:
+    #         decom_df[url_columns] = decom_df[url_columns].astype("string")
+    #     asin_columns = [i for i in decom_df.columns if 'asin' in i.lower()]
+    #     if len(asin_columns) > 0:
+    #         decom_df[asin_columns] = decom_df[asin_columns].astype("string")
+    #     if 'pending-quantity' in decom_df.columns:
+    #         decom_df['pending-quantity'] = decom_df['pending-quantity'].map(
+    #             lambda x: 0 if pd.isna(x) or np.isinf(x) else x).astype("int32")
+    #     deletecolumns = [i for i in decom_df.columns if 'zshop' in i.lower()]
+    #     decom_df.drop(columns=deletecolumns, inplace=True)
+    #     if 'quantity' in decom_df.columns:
+    #         decom_df['quantity'] = decom_df['quantity'].map(lambda x: 0 if pd.isna(x) or np.isinf(x) else x).astype(
+    #             "int32")
+    #     decom_df['opendate_date'] = decom_df['open-date'].map(lambda x: self.datetime_deal(x))
+    #     if 'add-delete' in decom_df.columns:
+    #         decom_df['add-delete'] = decom_df['add-delete'].astype('string', errors='ignore')
+    #     if 'will-ship-internationally' in decom_df.columns:
+    #         decom_df['will-ship-internationally'] = decom_df['will-ship-internationally'].astype('string',
+    #                                                                                              errors='ignore')
+    #     if 'expedited-shipping' in decom_df.columns:
+    #         decom_df['expedited-shipping'] = decom_df['expedited-shipping'].astype('string', errors='ignore')
+    #     decom_df['updateTime'] = datetime.now()
+    #     decom_df['timezone'] = "UTC"
+    #     decom_df['seller_id'] = seller_id
+    #     #
+    #     decom_df['item-description'] = decom_df['item-description'].str.slice(0, 500)
+    #     decom_df[decom_df.select_dtypes(float).columns] = decom_df[decom_df.select_dtypes(float).columns].fillna(0.0)
+    #     decom_df[decom_df.select_dtypes(int).columns] = decom_df[decom_df.select_dtypes(int).columns].fillna(0)
+    #     decom_df[decom_df.select_dtypes(datetime).columns] = decom_df[decom_df.select_dtypes(datetime).columns].astype(
+    #         'string')
+    #     decom_df.fillna('', inplace=True)
+    #     # print(decom_df.info())
+    #     return decom_df
+
+    def GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA(self,refresh_token,conn=None,seller_id=None,days=-1,**kwargs): # FBA库存信息
         try:
             para = {"reportType": ReportType.GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA}
             reportid = self.create_report(**para)
-
             df = self.decompression(reportid)
             if len(df)==0:
                 return pd.DataFrame()
@@ -316,36 +317,41 @@ class SpApiRequest:
         return df_rel
 
 
-
-    def GET_FLAT_FILE_OPEN_LISTINGS_DATA(self,refresh_token,conn=None,seller_id=None,days=-1): # To datatable asj_ads.seller_listings
+    def GET_FLAT_FILE_OPEN_LISTINGS_DATA(self,refresh_token,conn=None,seller_id=None,days=-1): # To datatable asj_ads.seller_listings- listing信息，包括fbm库存
         para = {"reportType": ReportType.GET_MERCHANT_LISTINGS_ALL_DATA}
         reportid = self.create_report(**para)
         df = self.decompression(reportid)
         if len(df)>0:
-            if self.marketplace.marketplace_id =='A1VC38T7YXB528':
+            if self.marketplace.marketplace_id =='A1VC38T7YXB528': # 该站点的数据列有所不同
                 df.columns = ['item-name','listing-id','seller-sku','price','quantity','open-date','product-id-type','item-description',
                               'item-condition','overseas shipping','fast shipping','asin1','stock_number','fulfillment-channel','merchant-shipping-group','status']
 
             df['seller_id'] = seller_id
             df['marketplace_id'] = self.marketplace.marketplace_id
             df['country_code'] = str(self.marketplace)[-2:]
-            if 'fulfilment-channel' in df.columns:
-                print("changed fulfilment-channel:")
-                print(seller_id,self.marketplace)
+            if 'fulfilment-channel' in df.columns: # 判断是否存在’fulfilment‘字段（1个film），如果存在则添加一个’fulfillment‘字段（两个fillm）
                 df['fulfillment-channel'] = df['fulfilment-channel'].copy()
+            # 如果是amazon，则字段改为FBA
             df['fulfillment_channel'] = df['fulfillment-channel'].map(lambda x:"FBA" if not pd.isna(x) and len(x)>0 and str(x)[1:4] in "AMAZON" else x)
+            # 如果是DEFAULT，则字段该为FBM
             df['fulfillment_channel'] = df['fulfillment_channel'].map(lambda x: "FBM" if not pd.isna(x) and len(x)>0 and str(x)[1:4] in "DEFAULT" else x)
 
-            if 'asin1' not in df.columns:
+            if 'asin1' not in df.columns: # 如果不存在asin1,则添加asin1字段
                 df['asin1'] = ''
-            if 'product-id' not in df.columns:
+            if 'product-id' not in df.columns: # 如果不存在product-id，则添加product-id字段
                 df['product-id'] = ''
 
             # 空值处理
             # df['quantity'] = df['quantity'].fillna(0).astype('int64',errors='ignore')
             df['quantity'] = df['quantity'].map(lambda x:0 if pd.isna(x)==True else int(x))
+
+            #库存数量格式处理
             df['quantity'] = df['quantity'].astype('int64')
+
+            # 填充NA值
             df[['listing-id','seller_id','asin1','seller-sku','country_code','marketplace_id','fulfillment_channel','status','product-id']] = df[['listing-id','seller_id','asin1','seller-sku','country_code','marketplace_id','fulfillment_channel','status','product-id']].fillna('').astype('string',errors='ignore')
+
+            # 为NA的价格填充0
             df['price'] = df['price'].fillna(0.0).astype('float64',errors='ignore')
             df.fillna('',inplace=True)
 
@@ -353,20 +359,27 @@ class SpApiRequest:
             df['opendate'] = df['open-date'].map(lambda x: self.datetime_deal(x))
             df['update_datetime'] = datetime.now(pytz.UTC).date()
 
+            # 保留列
             origin_columns = ['listing-id','seller_id',
                               'asin1','seller-sku','title','image_link','country_code',
                               'marketplace_id','quantity','fulfillment_channel',
                               'price','opendate','status','update_datetime','product-id','product-id-type','modifier'
                               ]
-            conn = SpApiRequest.mysql_connect_auth_lst()
+
+            # 连接数据库
+            conn = SpApiRequest.Data_auth()
             cursor = conn.cursor()
+            # 执行语句，筛选出asin不为空并且product_id不为空的两列唯一数据。
             cursor.execute("""select product_id,asin from (select * from asj_ads.seller_listings where asin is not null 
                                 and asin<>'' and product_id is not null and product_id <>'') t1 group by product_id,asin""")
             query_ = cursor.fetchall()
             col_name = [i[0] for i in cursor.description]
             df_datatable = pd.DataFrame(query_, columns=col_name)
+
+            # 合并数据，左表为新下载的数据，右表为数据库查询的数据
             merged_df = df.merge(df_datatable[['product_id','asin']],how='left',left_on='product-id',right_on='product_id')
 
+            # 功能函数，提取asin
             def func_(asin,asin1,product_id,cred,market_p,seller_id,sku):
                 if 'B0' in str(product_id)[:3]:
                     return str(product_id)
@@ -391,16 +404,22 @@ class SpApiRequest:
                     asin = r1.payload.get("summaries")[0].get("asin")
                     return asin
 
+            # 应用处理函数，返回asin
             merged_df['asin1'] = merged_df.apply(lambda x:func_(x['asin'],x['asin1'],x['product-id'],self.credentials,self.marketplace,seller_id,x['seller-sku']),axis=1) #x['asin'] if pd.isna(x['asin1']) or x['asin1']=='' else x['asin1']
             merged_df['image_link'] = ''
+
+            # 暂时将refresh_token添加在title列，后面获取asin详细数据时需要用到
             merged_df['title'] = refresh_token
             merged_df['modifier'] = ''
 
-
+            # 填充NA值
             merged_df.fillna('',inplace=True)
             df1 = merged_df.copy()
+
+            #获取FBA库存数据
             df_fbaInventory = self.GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA(refresh_token, conn, seller_id, days)
 
+            # 合并fba库存数据
             if len(df_fbaInventory)>0:
                 df1 = df1.merge(df_fbaInventory,how='left',left_on=['asin1','seller-sku','seller_id','marketplace_id','country_code'],right_on=['asin_','sku_','seller_id_','marketplace_id_','country_code_'])
                 df1['quantity'] = df1.apply(lambda x:x['afn-fulfillable-quantity'] if x['fulfillment_channel']=='FBA' or pd.isna(['afn-fulfillable-quantity'])==False else x['quantity'],axis=1)
@@ -408,13 +427,15 @@ class SpApiRequest:
             df1['quantity'] = df1['quantity'].map(lambda x:0 if pd.isna(x) else int(x))
             df1['quantity'] = df1['quantity'].fillna(0)
 
+            # 判断更新数据
             update_df = self.update_data(df1,seller_id,str(self.marketplace)[-2:],conn)
             if len(update_df)==0:
                 return '无更新数据插入'
             # update_df['country_code'] = update_df['country_code'].map({"GB":"UK"})
-            conn = SpApiRequest.mysql_connect_auth_lst()
+            conn = SpApiRequest.Data_auth()
             cursor = conn.cursor()
 
+            # 插入更新的数据
             try:
                 insertsql = """insert into
                 asj_ads.seller_listings(listing_id,seller_id,asin,sku,title,image_link,country_code,marketplace_id,quantity,
@@ -434,35 +455,35 @@ class SpApiRequest:
                 conn.rollback()
                 return '出错回滚'
 
-    def get_listing_info(self, sku,seller_id): # desprecated
-        listingClient = ListingsItems(credentials=self.credentials, marketplace=self.marketplace)
-        try:
-            r1 = listingClient.get_listings_item(sellerId=seller_id, sku=sku)
-            # print(r1.payload)
-            json_content = r1.payload.get("summaries")[0]
-            item_name = json_content.get("itemName")
-            item_name ='###' if item_name==None else item_name
-            img = json_content.get("mainImage")
-            img_url = '###' if img is None else img.get("link")
-            return str(img_url)+"-----"+ str(item_name)
-        except Exception as e:
-            try:
-                print("获取图片url过程错误重试, 错误message: ",e)
-                time.sleep(3)
-                r1 = listingClient.get_listings_item(sellerId=seller_id, sku=sku)
-                print(r1.payload)
-                json_content = r1.payload.get("summaries")[0]
-
-                item_name = json_content.get("itemName")
-                item_name = '###' if item_name == None else item_name
-                img = json_content.get("mainImage")
-                img_url = '###' if img is None else img.get("link")
-                return str(img_url)+"-----"+ str(item_name)
-            except Exception as e:
-                print(e)
-                return "###-----###"
-
-    def datetime_deal(self,timestring): # used in GET_FLAT_FILE_OPEN_LISTINGS_DATA, time deal
+    # def get_listing_info(self, sku,seller_id): # desprecated
+    #     listingClient = ListingsItems(credentials=self.credentials, marketplace=self.marketplace)
+    #     try:
+    #         r1 = listingClient.get_listings_item(sellerId=seller_id, sku=sku)
+    #         # print(r1.payload)
+    #         json_content = r1.payload.get("summaries")[0]
+    #         item_name = json_content.get("itemName")
+    #         item_name ='###' if item_name==None else item_name
+    #         img = json_content.get("mainImage")
+    #         img_url = '###' if img is None else img.get("link")
+    #         return str(img_url)+"-----"+ str(item_name)
+    #     except Exception as e:
+    #         try:
+    #             print("获取图片url过程错误重试, 错误message: ",e)
+    #             time.sleep(3)
+    #             r1 = listingClient.get_listings_item(sellerId=seller_id, sku=sku)
+    #             print(r1.payload)
+    #             json_content = r1.payload.get("summaries")[0]
+    #
+    #             item_name = json_content.get("itemName")
+    #             item_name = '###' if item_name == None else item_name
+    #             img = json_content.get("mainImage")
+    #             img_url = '###' if img is None else img.get("link")
+    #             return str(img_url)+"-----"+ str(item_name)
+    #         except Exception as e:
+    #             print(e)
+    #             return "###-----###"
+
+    def datetime_deal(self,timestring): # used in GET_FLAT_FILE_OPEN_LISTINGS_DATA, time deal -时间处理函数
         timezone_ = {"AEST":"Australia/Sydney","AEDT":"Australia/Sydney","PST":"America/Los_Angeles",
                     "PDT":"America/Los_Angeles","CST":"America/Chicago","CDT":"America/Chicago",
                     "MET":"MET","MEST":"MET","BST":"Europe/London","GMT":"GMT","CET":"CET",
@@ -503,7 +524,7 @@ class SpApiRequest:
                     return datetime(1999,12,31,0,0,0)
 
     def update_data(self,df,seller_id,country_code,conn): # used in GET_FLAT_FILE_OPEN_LISTINGS_DATA, data compare
-        conn = SpApiRequest.mysql_connect_auth_lst()
+        conn = SpApiRequest.Data_auth()
         cursor = conn.cursor()
         columns = ['listing-id', 'seller_id',
          'asin1', 'seller-sku', 'title', 'image_link', 'country_code',
@@ -517,7 +538,6 @@ class SpApiRequest:
 
         df_data = pd.DataFrame(columns=columns)
         delete_list = []
-
         marketplace_id = self.marketplace.marketplace_id
         try:
             cursor.execute(f"""select * from
@@ -580,7 +600,7 @@ class SpApiRequest:
 
 
 
-    def GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE(self,seller_id,days=-2): # not be used
+    def GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE(self,seller_id,days=-2): # not be used，退货报告，空数据
         shopReportday = (datetime.now() + timedelta(days=days)).strftime("%Y-%m-%d")
         # print(shopReportday)
         para = {"reportType": ReportType.GET_SELLER_FEEDBACK_DATA,
@@ -593,7 +613,7 @@ class SpApiRequest:
 
 
 
-    def GET_SALES_AND_TRAFFIC_REPORT(self, refresh_token,seller_id,days=-2,**kwargs): # To datatable asj_ads.SalesAndTrafficByAsin
+    def GET_SALES_AND_TRAFFIC_REPORT(self, refresh_token,seller_id,days=-2,**kwargs): # To datatable asj_ads.SalesAndTrafficByAsin，销售流量表
         # ,level:Literal["PARENT","CHILD","SKU"]="PARENT")
         level = "PARENT" if len(kwargs.get("level"))==0 else kwargs.get("level")
         countryCode = None if kwargs.get("countryCode")==None else kwargs.get("countryCode")
@@ -601,11 +621,11 @@ class SpApiRequest:
         shopReportday = (datetime.now() + timedelta(days=days)).strftime("%Y-%m-%d")
         print(shopReportday,countryCode,seller_id)
         try:
-            conn = self.mysql_connect_auth_lst()
+            conn = self.Data_auth()
             cursor = conn.cursor()
         except:
             time.sleep(5)
-            conn = self.mysql_connect_auth_lst()
+            conn = self.Data_auth()
             cursor = conn.cursor()
         if level == 'SKU':
             query_judge = f"""select count(*) from asj_ads.SalesAndTrafficByAsin where data_date='{shopReportday}' and countryCode='{countryCode}' and childAsin is not Null and sku is not Null"""
@@ -631,11 +651,11 @@ class SpApiRequest:
         # print(decom_df.columns[0])
         data_rel = self.sales_traffic_datadeal(decom_df.columns[0],seller_id,countryCode)
         try:
-            conn = self.mysql_connect_auth_lst()
+            conn = self.Data_auth()
             cursor = conn.cursor()
         except:
             time.sleep(5)
-            conn = self.mysql_connect_auth_lst()
+            conn = self.Data_auth()
             cursor = conn.cursor()
         # print(list(conn.query("select * from amz_sp_api.orderReport")))
         sql = f"""
@@ -739,16 +759,16 @@ class SpApiRequest:
         return data_list
 
 
-    def GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL(self, refresh_token,seller_id,days=-1,**kwargs): # To datatable asj_ads.orderReport_
+    def GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL(self, refresh_token,seller_id,days=-1,**kwargs): # To datatable asj_ads.orderReport_ - 获取订单报告
         countryCode = None if kwargs.get("countryCode")==None else kwargs.get("countryCode")
         shopReportday = (datetime.now() + timedelta(days=days)).strftime("%Y-%m-%d")
         print(shopReportday)
         try:
-            conn = self.mysql_connect_auth_lst()
+            conn = self.Data_auth()
             cursor = conn.cursor()
         except:
             time.sleep(5)
-            conn = self.mysql_connect_auth_lst()
+            conn = self.Data_auth()
             cursor = conn.cursor()
         query_judge = f"""select count(*) from asj_ads.orderReport_ where ReportDate='{shopReportday}' and country_code='{countryCode}'"""
         print(query_judge)
@@ -807,11 +827,11 @@ class SpApiRequest:
                            ]
         list_df = decom_df[reserve_columns].to_numpy().tolist()
         try:
-            conn = self.mysql_connect_auth_lst()
+            conn = self.Data_auth()
             cursor = conn.cursor()
         except:
             time.sleep(5)
-            conn = self.mysql_connect_auth_lst()
+            conn = self.Data_auth()
             cursor = conn.cursor()
         # print(list(conn.query("select * from amz_sp_api.orderReport")))
         sql = f"""
@@ -848,8 +868,8 @@ class SpApiRequest:
 
 
     @classmethod
-    def listing_infoTable(cls): # To datatable asj_ads.Goods
-        conn = SpApiRequest.mysql_connect_auth_lst()
+    def listing_infoTable(cls): # To datatable asj_ads.Goods - Goods表，包括排名，图片，父子关系
+        conn = SpApiRequest.Data_auth()
         cursor = conn.cursor()
         cursor.execute(f"""select seller_id,country_code,asin,title from asj_ads.seller_listings where title is not null and title <>'' and (seller_id,country_code,asin) not in (select seller_id,countryCode,asin from asj_ads.Goods where update_time>='{datetime.today().date()}') group by seller_id,title,country_code,asin order by country_code desc""")
         query_ = cursor.fetchall()
@@ -857,7 +877,7 @@ class SpApiRequest:
         col_name = [i[0] for i in cursor.description]
         df_datatable = pd.DataFrame(query_, columns=col_name)
         count=0
-        distance = 50
+        distance = 20
         print(len(df_datatable))
         while count<len(df_datatable):
             print(f"进度：{round(count / len(df_datatable) * 100, 2)}%")
@@ -865,11 +885,11 @@ class SpApiRequest:
             count = count+distance
             df['detail_info'] = df.apply(lambda x: cls.get_listing_info01(x['title'],x['country_code'],x['asin'],x['seller_id']),axis=1)
             detail_info_k = df['detail_info'].map(lambda x: list(x.keys())).to_numpy().tolist()
-            detail_info_v = df['detail_info'].map(lambda x: list(x.values())).to_numpy().tolist()
+            detail_info = df['detail_info'].to_numpy().tolist() #df['detail_info'].map(lambda x: list(x.values())).to_numpy().tolist()
 
-            conn = SpApiRequest.mysql_connect_auth_lst()
+            conn = SpApiRequest.Data_auth()
             print(count)
-            SpApiRequest.Goods_insert(conn,detail_info_v,detail_info_k)
+            SpApiRequest.Goods_insert(conn,detail_info,detail_info_k)
 
             if count%distance==0:
                 cursor = conn.cursor()
@@ -937,7 +957,7 @@ class SpApiRequest:
         return detail_info
 
     @staticmethod
-    def variations_judge(cate_item, asin): # used in listing_infoTable
+    def variations_judge(cate_item, asin): # used in listing_infoTable， 判断是否有父子关系
         try:
             variations = cate_item.get_catalog_item(asin=asin, **{"includedData": ['relationships']})#'variations',
             var_info = variations.payload
@@ -966,6 +986,7 @@ class SpApiRequest:
             return {"IsParent": IsParent, "parent_asin": parent_asin}
         except:
             try:
+                time.sleep(7.5)
                 variations = cate_item.get_catalog_item(asin=asin,
                                                         **{"includedData": ['relationships']})  # 'variations',
                 var_info = variations.payload
@@ -995,29 +1016,44 @@ class SpApiRequest:
                 return {"IsParent": 'Erro', "parent_asin": 'Erro'}
 
     @staticmethod
-    def Goods_insert(conn,detail_info_v,detail_info_k): # To datatable asj.Goods
+    def Goods_insert(conn,detail_info,detail_info_k): # To datatable asj.Goods
+        print(detail_info)
+        df = pd.DataFrame(detail_info)
+        print(df.columns)
         try:
             cursor = conn.cursor()
         except:
             time.sleep(2.5)
-            conn = SpApiRequest.mysql_connect_auth_lst()
+            conn = SpApiRequest.Data_auth()
             cursor = conn.cursor()
 
+        query_sql = "select * from asj_ads.Goods"
+        cursor.execute(query_sql)
+        col = [i[0] for i in cursor.description]
+        query_rel = cursor.fetchall()
+        df_query = pd.DataFrame(query_rel, columns=col)
+        merge_df = df.merge(df_query,how='left',on=['asin', 'countryCode', 'marketplace_id', 'seller_id'],suffixes=['','_right'])
+        merge_df['IsParent'] = merge_df.apply(lambda x:x['IsParent_right'] if x['IsParent']=='Erro' else x['IsParent'],axis=1)
+        merge_df['parent_asin'] = merge_df.apply(lambda x: x['parent_asin_right'] if x['parent_asin'] == 'Erro' else x['parent_asin'], axis=1)
+
+        detail_info_value = merge_df[['main_image', 'productTypes', 'BigCat_rank', 'BigCat_title', 'SmallCat_rank',
+                                     'SmallCat_title', 'brandName', 'browseNode', 'itemName', 'IsParent', 'parent_asin',
+                                     'asin', 'countryCode', 'marketplace_id', 'seller_id', 'update_time']].to_numpy().tolist()
         try:
             insertsql = """insert into
                         asj_ads.Goods(main_image, productTypes, BigCat_rank, BigCat_title, SmallCat_rank, SmallCat_title, brandName, browseNode, itemName, IsParent, parent_asin, asin, countryCode, marketplace_id, seller_id, update_time)
                         values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"""
             conn.begin()
-            cursor.executemany(insertsql, tuple(detail_info_v))
+            cursor.executemany(insertsql, tuple(detail_info_value))
             conn.commit()
             print("插入完成Goods")
-            # return '插入完成'
+
         except Exception as e:
             print("插入错误Goods：", e)
             conn.rollback()
 
         sales_rankData = []
-        for i in detail_info_v:
+        for i in detail_info_value:
             tmp_list = []
             for j in [2,3,4,5,6,8,11,12,13,14]:
                 tmp_list.extend([i[j]])
@@ -1038,7 +1074,7 @@ class SpApiRequest:
             conn.rollback()
 
     @staticmethod
-    def get_detail_cat(cate_item, asin, mak, countryCode): # used in listing_infoTable, category sp-api
+    def get_detail_cat(cate_item, asin, mak, countryCode): # used in listing_infoTable, category sp-api - 获取listing的信息数据
         try:
             detail_info = cate_item.get_catalog_item(asin=asin, **{
                 "includedData": ["images,productTypes,salesRanks,summaries"],
@@ -1082,7 +1118,7 @@ class SpApiRequest:
                     'brandName': '', 'browseNode': '', 'itemName': ''}
 
     @staticmethod
-    def data_judge_secondTry(refresh_token,sp_api,data_type,seller_id,auth_conn,days=-1,**kwargs): # Main-retry
+    def data_judge_secondTry(refresh_token,sp_api,data_type,seller_id,auth_conn,days=-1,**kwargs): # Main-retry， 重试函数，重试次数2次
         a_kw = kwargs
         try:
             SpApiRequest.data_judge(refresh_token,sp_api, data_type, seller_id, auth_conn,days=days,**a_kw)
@@ -1100,7 +1136,7 @@ class SpApiRequest:
                 SpApiRequest.data_judge(refresh_token, sp_api, data_type, seller_id, auth_conn, days=days, **a_kw)
 
     @staticmethod
-    def data_judge(refresh_token,sp_api,data_type,seller_id,auth_conn,days=-1,**kwargs): # select Report type
+    def data_judge(refresh_token,sp_api,data_type,seller_id,auth_conn,days=-1,**kwargs): # select Report type - 报告获取类型判断
         a_kw = kwargs
         if data_type == "GET_FLAT_FILE_OPEN_LISTINGS_DATA":
             return sp_api.GET_FLAT_FILE_OPEN_LISTINGS_DATA(refresh_token,auth_conn,seller_id,days)
@@ -1118,7 +1154,7 @@ class SpApiRequest:
             return ""
 
     @classmethod
-    def get_allShops(cls,data_type=Literal["GET_FLAT_FILE_OPEN_LISTINGS_DATA","GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL"],days=-1,**kwargs): # Main-AllCountries-AuthAndPost
+    def get_allShops(cls,data_type=Literal["GET_FLAT_FILE_OPEN_LISTINGS_DATA","GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL"],days=-1,**kwargs): # Main-AllCountries-AuthAndPost， 所有店铺数据报告获取的主要函数
         df = cls.auth_info()
         zosi = df.query("account_name=='AM-ZOSI-US'")['refresh_token'].to_numpy().tolist()
         # print(zosi)
@@ -1150,7 +1186,7 @@ class SpApiRequest:
                     # print(a_kw)
                     try:
                         print("refresh_token:",refresh_token,'data_type:',data_type,'seller_id:',seller_id,'marketplace:',marketplace.marketplace_id,'country_code:',a_kw['countryCode'],sep='\n')
-                        auth_conn = SpApiRequest.mysql_connect_auth()
+                        auth_conn = SpApiRequest.Token_auth()
                         # print(a_kw)
                         cls.data_judge_secondTry(refresh_token,sp_api, data_type, seller_id, auth_conn,days,**a_kw)
                         ## sp_api.GET_FLAT_FILE_OPEN_LISTINGS_DATA(auth_conn, seller_id)
@@ -1166,7 +1202,7 @@ class SpApiRequest:
                     sp_api = SpApiRequest(aws_credentials, marketplace)
                     a_kw['countryCode'] = str(marketplace)[-2:]
                     try:
-                        auth_conn = SpApiRequest.mysql_connect_auth()
+                        auth_conn = SpApiRequest.Token_auth()
                         cls.data_judge_secondTry(refresh_token,sp_api, data_type, seller_id, auth_conn,days,**a_kw)
                         ## sp_api.GET_FLAT_FILE_OPEN_LISTINGS_DATA(auth_conn, seller_id)
                     except Exception as e:
@@ -1175,7 +1211,7 @@ class SpApiRequest:
 
             else:
             # if region_circle not in ['NA','EU']:
-                auth_conn = SpApiRequest.mysql_connect_auth()
+                auth_conn = SpApiRequest.Token_auth()
                 print(region_circle)
                 marketplace = eval(f'Marketplaces.{region_circle}')
                 sp_api = SpApiRequest(aws_credentials, marketplace)
@@ -1187,7 +1223,9 @@ if __name__ == '__main__':
     # for days in range(35,45):
         # SpApiRequest.get_allShops("GET_SALES_AND_TRAFFIC_REPORT",days=-days,**{"level":"SKU"})
     # SpApiRequest.listing_infoTable()
-    # rel = SpApiRequest.get_catelog(account_name='ANLAPUS_US',country=Marketplaces.US,asin='B0BVXB4KT9')
+    # rel = SpApiRequest.get_catelog(account_name='AM-ZOSI-US',country=Marketplaces.US,asin='B0B8CPHSL4')
     # print(rel)
-    SpApiRequest.get_allShops("GET_FLAT_FILE_OPEN_LISTINGS_DATA")
-    pass
+    # SpApiRequest.get_allShops("GET_FLAT_FILE_OPEN_LISTINGS_DATA")
+    # pass
+    SpApiRequest.listing_infoTable()
+