add segment(profileid) for datatable

start_sync_amz_RightNowRun.py

@@ -95,8 +95,8 @@ if __name__ == '__main__':
     AWS_CREDENTIALS['refresh_token'] = refresh_token
     amz_report(conn, AWS_CREDENTIALS=AWS_CREDENTIALS)
 
-    # list_date = ["2023-11-21",]
-    # list_date = [f'2023-10-{"0"+str(i) if len(str(i))==1 else i}' for i in range(23,32)]
+    # list_date = ["2023-10-22",]
+    # # list_date = [f'2023-10-{"0"+str(i) if len(str(i))==1 else i}' for i in range(23,32)]
     # # print(list_date)
     # for date_ in list_date:
     #     print(date_)

sync_amz_data/public/sp_api_client.py

@@ -24,6 +24,7 @@ class SpApiRequest:
         self.marketplace = marketplace
         self.shopInfo = shop_infos(profile_id)
         self.timezone = self.shopInfo['time_zone']
+        self.profileid = profile_id
 
     def mysql_connect(self):
         conn = pymysql.connect(user="huangyifan",
@@ -132,29 +133,33 @@ class SpApiRequest:
             decom_df['expedited-shipping'] = decom_df['expedited-shipping'].astype('string',errors='ignore')
         decom_df['updateTime'] = datetime.now(tz=pytz.timezone(self.timezone))
         decom_df['timezone'] = self.timezone
+        decom_df['profileid'] = str(self.profileid)
         decom_df['item-description'] = decom_df['item-description'].str.slice(0,500)
         decom_df[decom_df.select_dtypes(float).columns] = decom_df[decom_df.select_dtypes(float).columns].fillna(0.0)
         decom_df[decom_df.select_dtypes(int).columns] = decom_df[decom_df.select_dtypes(int).columns].fillna(0)
         decom_df[decom_df.select_dtypes(datetime).columns] = decom_df[decom_df.select_dtypes(datetime).columns].astype(
             'string')
         decom_df.fillna('', inplace=True)
+        # print(decom_df.info())
         return decom_df
 
     def GET_MERCHANT_LISTINGS_ALL_DATA(self,limit=None):
+        start = time.time()
         para = {"reportType":ReportType.GET_MERCHANT_LISTINGS_ALL_DATA}
         reportid = self.create_report(**para)
         decom_df = self.decompression(reportid)
         print("连接数据库")
         conn = self.mysql_connect()
         cursor = conn.cursor()
-        bondary_date = (datetime.today() + timedelta(days=-28)).strftime("%Y-%m-%d")
+        bondary_date = (datetime.today()).strftime("%Y-%m-%d") #+ timedelta(days=-28)
         cursor.execute(f"""select * from amz_sp_api.productInfo where (mainImageUrl is not null and mainImageUrl not in ('', ' ')) and 
                         (`seller-sku` not in ('',' ') and `seller-sku` is not null) and 
                         `updateTime`>='{bondary_date}'""") #`seller-sku`,`updateTime`,`mainImageUrl`
         col = [i[0] for i in cursor.description]
         query_rel = cursor.fetchall()
-        print(query_rel[0])
+
         if len(query_rel)!=0:
+            print(query_rel[0])
             df = pd.DataFrame(query_rel,columns=col)
             listingid = df['listing-id'].to_numpy().tolist()
             decom_df = decom_df.query("`listing-id` not in @listingid")
@@ -179,8 +184,8 @@ class SpApiRequest:
             # print(list(conn.query("select * from amz_sp_api.orderReport")))
             sql = f"""
                         insert into amz_sp_api.productInfo
-                        values (%s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s)
-                    """
+                        values (%s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s,%s)
+                    """ #ok
             # print(sql)
             try:
                 conn.begin()
@@ -191,7 +196,10 @@ class SpApiRequest:
             except Exception as e:
                 conn.rollback()
                 print(e)
+                break
         print("全部完成")
+        end =time.time()
+        print("duration:",end-start)
         return decom_df
 
     def get_mainImage_url(self, sku):
@@ -225,6 +233,7 @@ class SpApiRequest:
         decom_df.fillna('',inplace=True)
         decom_df["ReportDate"] = parse(shopReportday)
         decom_df['timezone'] = self.timezone
+        decom_df['profileid'] = str(self.profileid)
         list_df = decom_df.to_numpy().tolist()
         print(list_df[0])
         # tuple_data = [tuple(i) for i in list_df]
@@ -233,8 +242,8 @@ class SpApiRequest:
         # print(list(conn.query("select * from amz_sp_api.orderReport")))
         sql = f"""
             insert into amz_sp_api.orderReport
-            values (%s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s)
-        """
+            values (%s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s,%s)
+        """ #ok
         # print(sql)
         try:
             conn.begin()
@@ -413,7 +422,7 @@ if __name__ == '__main__':
         'role_arn': 'arn:aws:iam::070880041373:role/Amazon_SP_API_ROLE'
     }
     sp_ = SpApiRequest(aws_credentials,Marketplaces.US,'3006125408623189')
-    sp_.GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL()
+    sp_.GET_MERCHANT_LISTINGS_ALL_DATA()
     """
     create database amz_sp_api;
     """