Merge branch 'yifan' of ASJ_ADS/sync_amz_data into master

start_sync_amz_RightNowRun.py

@@ -11,15 +11,15 @@ from sync_amz_data.DataTransform.Data_ETL import Common_ETLMethod,SP_ETL,SB_ETL,
 def amz_report(conn,AWS_CREDENTIALS,para=None):
     refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
     AWS_CREDENTIALS['refresh_token'] = refresh_token
-    # sb_report = SB_ETL(**AWS_CREDENTIALS)
+    sb_report = SB_ETL(**AWS_CREDENTIALS)
     # sb_report.reportV3_campaign_sbCampaigns_ETL(conn,params=para)
     # sb_report.reportV3_adGroup_sbAdGroup_ETL(conn,params=para)
     # sb_report.reportV3_sbCampaignPlacement_ETL(conn,params=para)
     # sb_report.reportV3_sbTargeting_ETL(conn,params=para)
     # sb_report.reportV3_sbSearchTerm_ETL(conn,params=para)
     # sb_report.reportV3_sbAds_ETL(conn, params=para)
-    # sb_report.reportV3_purchasedAsinRecord_ETL(conn,params=para)
-    #
+    sb_report.reportV3_purchasedAsinRecord_ETL(conn,params=para)
+    # #
     # sb_report.reportV2_campaignsRecord_ETL(conn,params=para)
     # sb_report.reportV2_campaignsVideo_ETL(conn,params=para)
     # sb_report.reportV2_adGroupsRecord_ETL(conn,params=para)
@@ -35,18 +35,18 @@ def amz_report(conn,AWS_CREDENTIALS,para=None):
     # sb_report.reportV2_targetsRecord_ETL(conn,params=para)
     # sb_report.reportV2_targetsVideo_ETL(conn,params=para)
 
-    sp_report = SP_ETL(**AWS_CREDENTIALS)
+    # sp_report = SP_ETL(**AWS_CREDENTIALS)
     # sp_report.reportV3_campaign_spCampaignsETL(conn,params=para)
     # sp_report.reportV3_adGroup_spCampaignsETL(conn,params=para)
     # sp_report.reportV3_campaignPlacement_spCampaignsETL(conn,params=para)
     # sp_report.reportV3_targeting_spTargetingETL(conn,params=para)
     # sp_report.reportV3_searchTerm_spSearchTermETL(conn,params=para)
     # sp_report.reportV3_advertiser_spAdvertisedProductETL(conn,params=para)
-    # para['Download']=True
+    ## para['Download']=True
+
+    # print('ok')
+    # sp_report.reportV3_asin_spPurchasedProductETL(conn,params=para)
 
-    print('ok')
-    sp_report.reportV3_asin_spPurchasedProductETL(conn,params=para)
-    # print(df)
     # sd_report = SD_ETL(**AWS_CREDENTIALS)
     #
     # sd_report.reportV3_campaign_sdCampaigns_ETL(conn,params=para)
@@ -91,19 +91,23 @@ if __name__ == '__main__':
 
     refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
     AWS_CREDENTIALS['refresh_token'] = refresh_token
-    # amz_report(conn, AWS_CREDENTIALS=AWS_CREDENTIALS)
+    amz_report(conn, AWS_CREDENTIALS=AWS_CREDENTIALS)
 
-    list_date = ["2024-03-18","2024-03-19","2024-03-20","2024-03-21",]
-    # list_date = [f'2024-03-{"0"+str(i) if len(str(i))==1 else i}' for i in range(31,32)]
-    print(list_date)
-    count = 1
-    for date_ in list_date:
-        # if count==1:
-        #     conn.command(f"delete from AmazonReport.SP_spPurchasedProduct_asinV3 where date='2024-03-18'")
-        print(date_)
-        print(date_.replace("-",""))
-        amz_report(conn,AWS_CREDENTIALS,para={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-        time.sleep(5)
-        count+=1
+    # list_date = ["2024-03-23",]
+    # list_date = [f'2024-04-{"0"+str(i) if len(str(i))==1 else i}' for i in range(10,11)]
+    # print(list_date)
+    # count = 1
+    # for date_ in list_date:
+    #     # if count==1:
+    #     #     conn.command(f"delete from AmazonReport.SD_advertiser_sdAdvertisedProductV3 where date='{date_}'")
+    #     #     conn.command(f"delete from AmazonReport.SD_asin_sdPurchasedProductV3 where date='{date_}'")
+    #     #conn.command(f"delete from AmazonReport.SB_sbPurchasedProduct_asinV3 where date='{date_}'")
+    #     print(date_)
+    #     print(date_.replace("-",""))
+    #     amz_report(conn,AWS_CREDENTIALS,para={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+    #     time.sleep(5)
+    #     count+=1
 
     conn.close()
+
+

sync_amz_data/public/sp_api_client.py

@@ -1020,6 +1020,7 @@ class SpApiRequest:
                 conn.rollback()
                 print("错误过程2",e)
 
+
     @staticmethod
     def get_listing_info01(refresh_token, countryCode, asin, seller_id): # used in listing_infoTable
         # print(refresh_token)
@@ -1125,7 +1126,8 @@ class SpApiRequest:
         merge_df = df.merge(df_query,how='left',on=['asin', 'countryCode', 'marketplace_id', 'seller_id'],suffixes=['','_right'])
         merge_df['IsParent'] = merge_df.apply(lambda x:x['IsParent_right'] if x['IsParent']=='Erro' else x['IsParent'],axis=1)
         merge_df['parent_asin'] = merge_df.apply(lambda x: x['parent_asin_right'] if x['parent_asin'] == 'Erro' else x['parent_asin'], axis=1)
-
+        erro_df = merge_df.query("IsParent=='Erro' or parent_asin=='Erro'")
+        merge_df.drop(erro_df.index,inplace=True)
         detail_info_value = merge_df[['main_image', 'productTypes', 'BigCat_rank', 'BigCat_title', 'SmallCat_rank',
                                      'SmallCat_title', 'brandName', 'browseNode', 'itemName', 'IsParent', 'parent_asin',
                                      'asin', 'countryCode', 'marketplace_id', 'seller_id', 'update_time']].to_numpy().tolist()