Merge branch 'yifan' of ASJ_ADS/sync_amz_data into master

start_sync_amz.py

@@ -9,7 +9,10 @@ from apscheduler.schedulers.blocking import BlockingScheduler
 from datetime import datetime,timedelta
 
 def amz_report(AWS_CREDENTIALS,para=None):
+    count = 1
     list_date = [((datetime.today()+timedelta(days=i)).date()).isoformat() for i in range(-2,-10,-1)]
+    conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    # conn.command('')
     if para is not None:
         list_date = [para['startDate']]
     # print(list_date)
@@ -17,6 +20,54 @@ def amz_report(AWS_CREDENTIALS,para=None):
         print(date_)
         print(date_.replace("-", ""))
         # amz_report(conn, AWS_CREDENTIALS, para={"startDate": date_, "endDate": date_, "date": date_.replace("-", "")})
+        if count==1:
+            delete_list = ['SB_adGroupsV2',
+                            'SB_adGroupsVideoV2',
+                            'SB_adsV2',
+                            'SB_adsVideoV2',
+                            'SB_campaignsPlacementV2',
+                            'SB_campaignsPlacementVideoV2',
+                            'SB_campaignsV2',
+                            'SB_campaignsVideoV2',
+                            'SB_keywordsQueryV2',
+                            'SB_keywordsQueryVideoV2',
+                            'SB_keywordsV2',
+                            'SB_keywordsVideoV2',
+                            'SB_sbAdGroup_adGroupV3',
+                            'SB_sbAds_adsV3',
+                            'SB_sbCampaigns_campaignV3',
+                            'SB_sbCampaigns_placementV3',
+                            'SB_sbPurchasedProduct_asinV3',
+                            'SB_sbSearchTerm_searchTermV3',
+                            'SB_sbTargeting_targetingV3',
+                            'SB_targetsV2',
+                            'SB_targetsVideoV2',
+                            'SD_adGroupsMatchedTargetV2',
+                            'SD_adGroupsV2',
+                            'SD_adsV2',
+                            'SD_advertiser_sdAdvertisedProductV3',
+                            'SD_asin_sdPurchasedProductV3',
+                            'SD_asinsV2',
+                            'SD_campaignsMatchedTargetV2',
+                            'SD_campaignsV2',
+                            'SD_sdAdGroup_adGroupMatchedTargetV3',
+                            'SD_sdAdGroup_adGroupV3',
+                            'SD_sdCampaigns_campaignMatchedTargetV3',
+                            'SD_sdCampaigns_campaignV3',
+                            'SD_targeting_sdTargetingMatchedTargetV3',
+                            'SD_targeting_sdTargetingV3',
+                            'SD_targetsMatchedTargetV2',
+                            'SD_targetsV2',
+                            'SP_spAdvertisedProduct_advertiserV3',
+                            'SP_spCampaigns_adGroupV3',
+                            'SP_spCampaigns_campaignV3',
+                            'SP_spCampaigns_placementV3',
+                            'SP_spPurchasedProduct_asinV3',
+                            'SP_spSearchTerm_searchTermV3',
+                            'SP_spTargeting_targetingV3']
+            for i in delete_list:
+                conn.command(f"delete from AmazonReport.{i} where date='{list_date[-2]}'")
+            count+=1
 
         try:
             refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']

start_sync_amz_RightNowRun.py

@@ -11,49 +11,52 @@ from sync_amz_data.DataTransform.Data_ETL import Common_ETLMethod,SP_ETL,SB_ETL,
 def amz_report(conn,AWS_CREDENTIALS,para=None):
     refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
     AWS_CREDENTIALS['refresh_token'] = refresh_token
-    sb_report = SB_ETL(**AWS_CREDENTIALS)
-    sb_report.reportV3_campaign_sbCampaigns_ETL(conn,params=para)
-    sb_report.reportV3_adGroup_sbAdGroup_ETL(conn,params=para)
-    sb_report.reportV3_sbCampaignPlacement_ETL(conn,params=para)
-    sb_report.reportV3_sbTargeting_ETL(conn,params=para)
-    sb_report.reportV3_sbSearchTerm_ETL(conn,params=para)
-    sb_report.reportV3_sbAds_ETL(conn, params=para)
-    sb_report.reportV3_purchasedAsinRecord_ETL(conn,params=para)
-
-    sb_report.reportV2_campaignsRecord_ETL(conn,params=para)
-    sb_report.reportV2_campaignsVideo_ETL(conn,params=para)
-    sb_report.reportV2_adGroupsRecord_ETL(conn,params=para)
-    sb_report.reportV2_adGroupsVideo_ETL(conn,params=para)
-    sb_report.reportV2_adsRecord_ETL(conn,params=para)
-    sb_report.reportV2_adsVideo_ETL(conn,params=para)
-    sb_report.reportV2_keywordsRecord_ETL(conn,params=para)
-    sb_report.reportV2_keywordsVideo_ETL(conn,params=para)
-    sb_report.reportV2_placementRecord_ETL(conn,params=para)
-    sb_report.reportV2_placementVideo_ETL(conn,params=para)
-    sb_report.reportV2_searchtermsRecord_ETL(conn,params=para)
-    sb_report.reportV2_searchtermsVideo_ETL(conn,params=para)
-    sb_report.reportV2_targetsRecord_ETL(conn,params=para)
-    sb_report.reportV2_targetsVideo_ETL(conn,params=para)
+    # sb_report = SB_ETL(**AWS_CREDENTIALS)
+    # sb_report.reportV3_campaign_sbCampaigns_ETL(conn,params=para)
+    # sb_report.reportV3_adGroup_sbAdGroup_ETL(conn,params=para)
+    # sb_report.reportV3_sbCampaignPlacement_ETL(conn,params=para)
+    # sb_report.reportV3_sbTargeting_ETL(conn,params=para)
+    # sb_report.reportV3_sbSearchTerm_ETL(conn,params=para)
+    # sb_report.reportV3_sbAds_ETL(conn, params=para)
+    # sb_report.reportV3_purchasedAsinRecord_ETL(conn,params=para)
+    #
+    # sb_report.reportV2_campaignsRecord_ETL(conn,params=para)
+    # sb_report.reportV2_campaignsVideo_ETL(conn,params=para)
+    # sb_report.reportV2_adGroupsRecord_ETL(conn,params=para)
+    # sb_report.reportV2_adGroupsVideo_ETL(conn,params=para)
+    # sb_report.reportV2_adsRecord_ETL(conn,params=para)
+    # sb_report.reportV2_adsVideo_ETL(conn,params=para)
+    # sb_report.reportV2_keywordsRecord_ETL(conn,params=para)
+    # sb_report.reportV2_keywordsVideo_ETL(conn,params=para)
+    # sb_report.reportV2_placementRecord_ETL(conn,params=para)
+    # sb_report.reportV2_placementVideo_ETL(conn,params=para)
+    # sb_report.reportV2_searchtermsRecord_ETL(conn,params=para)
+    # sb_report.reportV2_searchtermsVideo_ETL(conn,params=para)
+    # sb_report.reportV2_targetsRecord_ETL(conn,params=para)
+    # sb_report.reportV2_targetsVideo_ETL(conn,params=para)
 
     sp_report = SP_ETL(**AWS_CREDENTIALS)
-    sp_report.reportV3_campaign_spCampaignsETL(conn,params=para)
-    sp_report.reportV3_adGroup_spCampaignsETL(conn,params=para)
-    sp_report.reportV3_campaignPlacement_spCampaignsETL(conn,params=para)
-    sp_report.reportV3_targeting_spTargetingETL(conn,params=para)
-    sp_report.reportV3_searchTerm_spSearchTermETL(conn,params=para)
-    sp_report.reportV3_advertiser_spAdvertisedProductETL(conn,params=para)
-    sp_report.reportV3_asin_spPurchasedProductETL(conn,params=para)
-
-    sd_report = SD_ETL(**AWS_CREDENTIALS)
+    # sp_report.reportV3_campaign_spCampaignsETL(conn,params=para)
+    # sp_report.reportV3_adGroup_spCampaignsETL(conn,params=para)
+    # sp_report.reportV3_campaignPlacement_spCampaignsETL(conn,params=para)
+    # sp_report.reportV3_targeting_spTargetingETL(conn,params=para)
+    # sp_report.reportV3_searchTerm_spSearchTermETL(conn,params=para)
+    # sp_report.reportV3_advertiser_spAdvertisedProductETL(conn,params=para)
+    # para['Download']=True
 
-    sd_report.reportV3_campaign_sdCampaigns_ETL(conn,params=para)
-    sd_report.reportV3_campaignMT_sdCampaigns_ETL(conn,params=para)
-    sd_report.reportV3_adgroup_sdAdGroup_ETL(conn,params=para)
-    sd_report.reportV3_adgroupMT_sdAdGroup_ETL(conn,params=para)
-    sd_report.reportV3_targeting_sdTargeting_ETL(conn,params=para)
-    sd_report.reportV3_targetingMT_sdTargeting_ETL(conn,params=para)
-    sd_report.reportV3_asin_sdPurchasedProduct_ETL(conn,params=para)
-    sd_report.reportV3_advertiser_sdAdvertisedProduct_ETL(conn,params=para)
+    print('ok')
+    sp_report.reportV3_asin_spPurchasedProductETL(conn,params=para)
+    # print(df)
+    # sd_report = SD_ETL(**AWS_CREDENTIALS)
+    #
+    # sd_report.reportV3_campaign_sdCampaigns_ETL(conn,params=para)
+    # sd_report.reportV3_campaignMT_sdCampaigns_ETL(conn,params=para)
+    # sd_report.reportV3_adgroup_sdAdGroup_ETL(conn,params=para)
+    # sd_report.reportV3_adgroupMT_sdAdGroup_ETL(conn,params=para)
+    # sd_report.reportV3_targeting_sdTargeting_ETL(conn,params=para)
+    # sd_report.reportV3_targetingMT_sdTargeting_ETL(conn,params=para)
+    # sd_report.reportV3_asin_sdPurchasedProduct_ETL(conn,params=para)
+    # sd_report.reportV3_advertiser_sdAdvertisedProduct_ETL(conn,params=para)
 
     """
     # sd_report.reportV2_campaignsRecord_t2_ETL(conn,params=para)
@@ -90,14 +93,17 @@ if __name__ == '__main__':
     AWS_CREDENTIALS['refresh_token'] = refresh_token
     # amz_report(conn, AWS_CREDENTIALS=AWS_CREDENTIALS)
 
-    # list_date = ["2024-03-01","2024-03-24"]
-    list_date = [f'2024-03-{"0"+str(i) if len(str(i))==1 else i}' for i in range(31,32)]
-    # print(list_date)
+    list_date = ["2024-03-18","2024-03-19","2024-03-20","2024-03-21",]
+    # list_date = [f'2024-03-{"0"+str(i) if len(str(i))==1 else i}' for i in range(31,32)]
+    print(list_date)
+    count = 1
     for date_ in list_date:
+        # if count==1:
+        #     conn.command(f"delete from AmazonReport.SP_spPurchasedProduct_asinV3 where date='2024-03-18'")
         print(date_)
         print(date_.replace("-",""))
         amz_report(conn,AWS_CREDENTIALS,para={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
         time.sleep(5)
-
+        count+=1
 
     conn.close()

sync_amz_data/DataTransform/Data_ETL.py

@@ -506,10 +506,11 @@ class SP_ETL(SPClient, Common_ETLMethod):
         params = self.config_params(params)
         print("func_name:", "reportV3_asin_spPurchasedProductETL", '\n', "table_name:",
               "SP_spPurchasedProduct_asinV3")
-        if len(conn.query_df(
-                f"select * from AmazonReport.SP_spPurchasedProduct_asinV3 where date='{params['startDate']}'")) > 0:
-            logging.info("数据已存在...")
-            return 'Pass'
+        if params.get('Download') in [None,False]:
+            if len(conn.query_df(
+                    f"select * from AmazonReport.SP_spPurchasedProduct_asinV3 where date='{params['startDate']}'")) > 0:
+                logging.info("数据已存在...")
+                return 'Pass'
         params['reportType'] = "spPurchasedProduct"
         params['columns'] = [
             'date', 'portfolioId', 'campaignName', 'campaignId', 'adGroupName', 'adGroupId', 'keywordId', 'keyword',
@@ -536,9 +537,10 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report['advertisedSku'] = df_report['advertisedSku'].map(lambda x: str(x) if x is not None else '-')
         print(df_report.dtypes)
         # df_report.to_excel('ATEST.xlsx')
-        conn.insert_df("AmazonReport.SP_spPurchasedProduct_asinV3", df_report[params['columns']])
-        time.sleep(0.05)
-        print("插入完成SP_spPurchasedProduct_asinV3")
+        if params.get('Download') in [None, False]:
+            conn.insert_df("AmazonReport.SP_spPurchasedProduct_asinV3", df_report[params['columns']])
+            time.sleep(0.05)
+            print("插入完成SP_spPurchasedProduct_asinV3")
 
 
         df_rel =  df_report[params['columns']]

sync_amz_data/public/sp_api_client.py

@@ -644,6 +644,7 @@ class SpApiRequest:
         level = "PARENT" if len(kwargs.get("level"))==0 else kwargs.get("level")
         countryCode = None if kwargs.get("countryCode")==None else kwargs.get("countryCode")
         # print(level)
+
         shopReportday = (datetime.now() + timedelta(days=days)).strftime("%Y-%m-%d")
         print(shopReportday,countryCode,seller_id)
         try:
@@ -654,11 +655,11 @@ class SpApiRequest:
             conn = self.Data_auth()
             cursor = conn.cursor()
         if level == 'SKU':
-            query_judge = f"""select count(*) from asj_ads.SalesAndTrafficByAsin where data_date='{shopReportday}' and countryCode='{countryCode}' and childAsin is not Null and sku is not Null"""
+            query_judge = f"""select count(*) from asj_ads.SalesAndTrafficByAsin where seller_id='{seller_id}' and data_date='{shopReportday}' and countryCode='{countryCode}' and childAsin is not Null and sku is not Null"""
         elif level == 'CHILD':
-            query_judge = f"""select count(*) from asj_ads.SalesAndTrafficByAsin where data_date='{shopReportday}' and countryCode='{countryCode}' and sku is null and childAsin is not null"""
+            query_judge = f"""select count(*) from asj_ads.SalesAndTrafficByAsin where seller_id='{seller_id}' and data_date='{shopReportday}' and countryCode='{countryCode}' and sku is null and childAsin is not null"""
         elif level == 'PARENT':
-            query_judge = f"""select count(*) from asj_ads.SalesAndTrafficByAsin where data_date='{shopReportday}' and countryCode='{countryCode}' and sku is null and childAsin is null"""
+            query_judge = f"""select count(*) from asj_ads.SalesAndTrafficByAsin where seller_id='{seller_id}' and data_date='{shopReportday}' and countryCode='{countryCode}' and sku is null and childAsin is null"""
         else:
             return ''
         print(query_judge)

sync_get_order_data.py

@@ -16,7 +16,7 @@ def func_run():
         print(e)
     try:
         count = 0
-        for days in range(-2,-10,-1): #range(-63,-2,1): #
+        for days in range(-2,-10,-1): #range(-72,-2,1): #
             if count in (0,1,2):#[i for i in range(0,32)]:
                 conn = sp_api_client.SpApiRequest.Data_auth()
                 cursor = conn.cursor()