Merge branch 'yifan' of ASJ_ADS/sync_amz_data into master

start_sync_amz.py

@@ -1,5 +1,7 @@
 import time
 import pandas as pd
+from retry import retry
+
 from sync_amz_data.settings import LOG_CONF
 import logging.config
 logging.config.dictConfig(LOG_CONF)
@@ -273,7 +275,7 @@ def pre_func(AWS_CREDENTIALS,date_):
 
 def amz_report(AWS_CREDENTIALS,para=None):
     count = 1
-    list_date = [((datetime.today()+timedelta(days=i)).date()).isoformat() for i in range(-2,-10,-1)]
+    list_date = [((datetime.today()+timedelta(days=i)).date()).isoformat() for i in range(-2,-20,-1)]
     conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
     # conn.command('')
     if para is not None:
@@ -302,6 +304,7 @@ def amz_report(AWS_CREDENTIALS,para=None):
                    'SP_spPurchasedProduct_asinV3',
                    'SP_spSearchTerm_searchTermV3',
                    'SP_spTargeting_targetingV3']
+
     for date_ in list_date:
         print(date_)
         print(date_.replace("-", ""))
@@ -319,47 +322,86 @@ def amz_report(AWS_CREDENTIALS,para=None):
             print(e)
 
         pre_func(AWS_CREDENTIALS,date_)
-
-    conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
-    date_list = []
-    for table_name in delete_list:
-        try:
-            locals()[table_name] = conn.query_df(
-                f"select date,SUM(impressions) as impressions from AmazonReport.`{table_name}` group by date,profileId")
-        except:
+    try:
+        conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+        date_list = []
+        for table_name in delete_list:
             try:
                 locals()[table_name] = conn.query_df(
-                    f"select date,SUM(salesOtherSku1d) as salesOtherSku1d from AmazonReport.`{table_name}` group by date,profileId")
+                    f"select date,SUM(impressions) as impressions from AmazonReport.`{table_name}` group by date,profileId")
             except:
                 try:
                     locals()[table_name] = conn.query_df(
-                        f"select date,SUM(salesBrandHalo) as salesBrandHalo from AmazonReport.`{table_name}` group by date,profileId")
+                        f"select date,SUM(salesOtherSku1d) as salesOtherSku1d from AmazonReport.`{table_name}` group by date,profileId")
                 except:
-                    locals()[table_name] = conn.query_df(
-                        f"select date,SUM(sales14d) as sales14d  from AmazonReport.`{table_name}` group by date,profileId")
-        locals()[table_name].columns = ['date', 'refer_']
-        locals()[table_name]['refer_'] = locals()[table_name]['refer_'].astype('float64')
-        todayMinus2 = (datetime.today().date() + timedelta(days=-2))
-        reback_21days = (todayMinus2 + timedelta(days=-23)).isoformat()
-        df = pd.date_range(start=reback_21days, end=todayMinus2.isoformat(), freq='1d')
-        df1 = pd.DataFrame(df, columns=['date_'])
-        merge_df = df1.merge(locals()[table_name], left_on=['date_'], right_on=['date'], how='left')
-        rel = merge_df.query("date_!=date or refer_<1 or refer_.isna()")['date_'].map(
-            lambda x: x.date().isoformat()).tolist()
-        print(rel,date_list)
-        if len(rel)>0:
-            for date in rel:
-                date_list.extend([date])
-                conn.command(
-                    f"delete from AmazonReport.`{table_name}` where date='{date}' and profileId='{AWS_CREDENTIALS['profile_id']}'")
-    if len(date_list)>0:
-        print(list(set(date_list)))
-        for date_retry in list(set(date_list)):
+                    try:
+                        locals()[table_name] = conn.query_df(
+                            f"select date,SUM(salesBrandHalo) as salesBrandHalo from AmazonReport.`{table_name}` group by date,profileId")
+                    except:
+                        locals()[table_name] = conn.query_df(
+                            f"select date,SUM(sales14d) as sales14d  from AmazonReport.`{table_name}` group by date,profileId")
+            locals()[table_name].columns = ['date', 'refer_']
+            locals()[table_name]['refer_'] = locals()[table_name]['refer_'].astype('float64')
+            todayMinus2 = (datetime.today().date() + timedelta(days=-2))
+            reback_21days = (todayMinus2 + timedelta(days=-23)).isoformat()
+            df = pd.date_range(start=reback_21days, end=todayMinus2.isoformat(), freq='1d')
+            df1 = pd.DataFrame(df, columns=['date_'])
+            merge_df = df1.merge(locals()[table_name], left_on=['date_'], right_on=['date'], how='left')
+            rel = merge_df.query("date_!=date or refer_<1 or refer_.isna()")['date_'].map(
+                lambda x: x.date().isoformat()).tolist()
+            print(rel,date_list)
+            if len(rel)>0:
+                for date in rel:
+                    date_list.extend([date])
+                    conn.command(
+                        f"delete from AmazonReport.`{table_name}` where date='{date}' and profileId='{AWS_CREDENTIALS['profile_id']}'")
+        if len(date_list)>0:
+            print(list(set(date_list)))
+            for date_retry in list(set(date_list)):
+                try:
+                    pre_func(AWS_CREDENTIALS, date_retry)
+                except:
+                    pre_func(AWS_CREDENTIALS, date_retry)
+    except:
+        conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+        date_list = []
+        for table_name in delete_list:
             try:
-                pre_func(AWS_CREDENTIALS, date_retry)
+                locals()[table_name] = conn.query_df(
+                    f"select date,SUM(impressions) as impressions from AmazonReport.`{table_name}` group by date,profileId")
             except:
-                pre_func(AWS_CREDENTIALS, date_retry)
-
+                try:
+                    locals()[table_name] = conn.query_df(
+                        f"select date,SUM(salesOtherSku1d) as salesOtherSku1d from AmazonReport.`{table_name}` group by date,profileId")
+                except:
+                    try:
+                        locals()[table_name] = conn.query_df(
+                            f"select date,SUM(salesBrandHalo) as salesBrandHalo from AmazonReport.`{table_name}` group by date,profileId")
+                    except:
+                        locals()[table_name] = conn.query_df(
+                            f"select date,SUM(sales14d) as sales14d  from AmazonReport.`{table_name}` group by date,profileId")
+            locals()[table_name].columns = ['date', 'refer_']
+            locals()[table_name]['refer_'] = locals()[table_name]['refer_'].astype('float64')
+            todayMinus2 = (datetime.today().date() + timedelta(days=-2))
+            reback_21days = (todayMinus2 + timedelta(days=-23)).isoformat()
+            df = pd.date_range(start=reback_21days, end=todayMinus2.isoformat(), freq='1d')
+            df1 = pd.DataFrame(df, columns=['date_'])
+            merge_df = df1.merge(locals()[table_name], left_on=['date_'], right_on=['date'], how='left')
+            rel = merge_df.query("date_!=date or refer_<1 or refer_.isna()")['date_'].map(
+                lambda x: x.date().isoformat()).tolist()
+            print(rel, date_list)
+            if len(rel) > 0:
+                for date in rel:
+                    date_list.extend([date])
+                    conn.command(
+                        f"delete from AmazonReport.`{table_name}` where date='{date}' and profileId='{AWS_CREDENTIALS['profile_id']}'")
+        if len(date_list) > 0:
+            print(list(set(date_list)))
+            for date_retry in list(set(date_list)):
+                try:
+                    pre_func(AWS_CREDENTIALS, date_retry)
+                except:
+                    pre_func(AWS_CREDENTIALS, date_retry)
 
 if __name__ == '__main__':
     # AccountTask("3006125408623189").do({"record": "portfolios"})

start_sync_amz_RightNowRun.py

@@ -12,51 +12,51 @@ def amz_report(conn,AWS_CREDENTIALS,para=None):
     refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
     AWS_CREDENTIALS['refresh_token'] = refresh_token
     sb_report = SB_ETL(**AWS_CREDENTIALS)
-    # sb_report.reportV3_campaign_sbCampaigns_ETL(conn,params=para)
-    # sb_report.reportV3_adGroup_sbAdGroup_ETL(conn,params=para)
-    # sb_report.reportV3_sbCampaignPlacement_ETL(conn,params=para)
-    # sb_report.reportV3_sbTargeting_ETL(conn,params=para)
-    # sb_report.reportV3_sbSearchTerm_ETL(conn,params=para)
-    # sb_report.reportV3_sbAds_ETL(conn, params=para)
+    sb_report.reportV3_campaign_sbCampaigns_ETL(conn,params=para)
+    sb_report.reportV3_adGroup_sbAdGroup_ETL(conn,params=para)
+    sb_report.reportV3_sbCampaignPlacement_ETL(conn,params=para)
+    sb_report.reportV3_sbTargeting_ETL(conn,params=para)
+    sb_report.reportV3_sbSearchTerm_ETL(conn,params=para)
+    sb_report.reportV3_sbAds_ETL(conn, params=para)
     sb_report.reportV3_purchasedAsinRecord_ETL(conn,params=para)
     # #
-    # sb_report.reportV2_campaignsRecord_ETL(conn,params=para)
-    # sb_report.reportV2_campaignsVideo_ETL(conn,params=para)
-    # sb_report.reportV2_adGroupsRecord_ETL(conn,params=para)
-    # sb_report.reportV2_adGroupsVideo_ETL(conn,params=para)
-    # sb_report.reportV2_adsRecord_ETL(conn,params=para)
-    # sb_report.reportV2_adsVideo_ETL(conn,params=para)
-    # sb_report.reportV2_keywordsRecord_ETL(conn,params=para)
-    # sb_report.reportV2_keywordsVideo_ETL(conn,params=para)
-    # sb_report.reportV2_placementRecord_ETL(conn,params=para)
-    # sb_report.reportV2_placementVideo_ETL(conn,params=para)
-    # sb_report.reportV2_searchtermsRecord_ETL(conn,params=para)
-    # sb_report.reportV2_searchtermsVideo_ETL(conn,params=para)
-    # sb_report.reportV2_targetsRecord_ETL(conn,params=para)
-    # sb_report.reportV2_targetsVideo_ETL(conn,params=para)
+    sb_report.reportV2_campaignsRecord_ETL(conn,params=para)
+    sb_report.reportV2_campaignsVideo_ETL(conn,params=para)
+    sb_report.reportV2_adGroupsRecord_ETL(conn,params=para)
+    sb_report.reportV2_adGroupsVideo_ETL(conn,params=para)
+    sb_report.reportV2_adsRecord_ETL(conn,params=para)
+    sb_report.reportV2_adsVideo_ETL(conn,params=para)
+    sb_report.reportV2_keywordsRecord_ETL(conn,params=para)
+    sb_report.reportV2_keywordsVideo_ETL(conn,params=para)
+    sb_report.reportV2_placementRecord_ETL(conn,params=para)
+    sb_report.reportV2_placementVideo_ETL(conn,params=para)
+    sb_report.reportV2_searchtermsRecord_ETL(conn,params=para)
+    sb_report.reportV2_searchtermsVideo_ETL(conn,params=para)
+    sb_report.reportV2_targetsRecord_ETL(conn,params=para)
+    sb_report.reportV2_targetsVideo_ETL(conn,params=para)
 
-    # sp_report = SP_ETL(**AWS_CREDENTIALS)
-    # sp_report.reportV3_campaign_spCampaignsETL(conn,params=para)
-    # sp_report.reportV3_adGroup_spCampaignsETL(conn,params=para)
-    # sp_report.reportV3_campaignPlacement_spCampaignsETL(conn,params=para)
-    # sp_report.reportV3_targeting_spTargetingETL(conn,params=para)
-    # sp_report.reportV3_searchTerm_spSearchTermETL(conn,params=para)
-    # sp_report.reportV3_advertiser_spAdvertisedProductETL(conn,params=para)
+    sp_report = SP_ETL(**AWS_CREDENTIALS)
+    sp_report.reportV3_campaign_spCampaignsETL(conn,params=para)
+    sp_report.reportV3_adGroup_spCampaignsETL(conn,params=para)
+    sp_report.reportV3_campaignPlacement_spCampaignsETL(conn,params=para)
+    sp_report.reportV3_targeting_spTargetingETL(conn,params=para)
+    sp_report.reportV3_searchTerm_spSearchTermETL(conn,params=para)
+    sp_report.reportV3_advertiser_spAdvertisedProductETL(conn,params=para)
     ## para['Download']=True
 
     # print('ok')
-    # sp_report.reportV3_asin_spPurchasedProductETL(conn,params=para)
+    sp_report.reportV3_asin_spPurchasedProductETL(conn,params=para)
 
-    # sd_report = SD_ETL(**AWS_CREDENTIALS)
-    #
-    # sd_report.reportV3_campaign_sdCampaigns_ETL(conn,params=para)
-    # sd_report.reportV3_campaignMT_sdCampaigns_ETL(conn,params=para)
-    # sd_report.reportV3_adgroup_sdAdGroup_ETL(conn,params=para)
-    # sd_report.reportV3_adgroupMT_sdAdGroup_ETL(conn,params=para)
-    # sd_report.reportV3_targeting_sdTargeting_ETL(conn,params=para)
-    # sd_report.reportV3_targetingMT_sdTargeting_ETL(conn,params=para)
-    # sd_report.reportV3_asin_sdPurchasedProduct_ETL(conn,params=para)
-    # sd_report.reportV3_advertiser_sdAdvertisedProduct_ETL(conn,params=para)
+    sd_report = SD_ETL(**AWS_CREDENTIALS)
+
+    sd_report.reportV3_campaign_sdCampaigns_ETL(conn,params=para)
+    sd_report.reportV3_campaignMT_sdCampaigns_ETL(conn,params=para)
+    sd_report.reportV3_adgroup_sdAdGroup_ETL(conn,params=para)
+    sd_report.reportV3_adgroupMT_sdAdGroup_ETL(conn,params=para)
+    sd_report.reportV3_targeting_sdTargeting_ETL(conn,params=para)
+    sd_report.reportV3_targetingMT_sdTargeting_ETL(conn,params=para)
+    sd_report.reportV3_asin_sdPurchasedProduct_ETL(conn,params=para)
+    sd_report.reportV3_advertiser_sdAdvertisedProduct_ETL(conn,params=para)
 
     """
     # sd_report.reportV2_campaignsRecord_t2_ETL(conn,params=para)
@@ -88,25 +88,32 @@ if __name__ == '__main__':
     conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
     timezone_ = Common_ETLMethod(**AWS_CREDENTIALS).timeZone()
     print(timezone_)
+    print("======================")
 
     refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
     AWS_CREDENTIALS['refresh_token'] = refresh_token
-    amz_report(conn, AWS_CREDENTIALS=AWS_CREDENTIALS)
+    # amz_report(conn, AWS_CREDENTIALS=AWS_CREDENTIALS)
 
     # list_date = ["2024-03-23",]
-    # list_date = [f'2024-04-{"0"+str(i) if len(str(i))==1 else i}' for i in range(10,11)]
-    # print(list_date)
-    # count = 1
-    # for date_ in list_date:
-    #     # if count==1:
-    #     #     conn.command(f"delete from AmazonReport.SD_advertiser_sdAdvertisedProductV3 where date='{date_}'")
-    #     #     conn.command(f"delete from AmazonReport.SD_asin_sdPurchasedProductV3 where date='{date_}'")
-    #     #conn.command(f"delete from AmazonReport.SB_sbPurchasedProduct_asinV3 where date='{date_}'")
-    #     print(date_)
-    #     print(date_.replace("-",""))
-    #     amz_report(conn,AWS_CREDENTIALS,para={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
-    #     time.sleep(5)
-    #     count+=1
+    # print("11111111111111111111111111111")
+    list_date = [f'2024-07-{"0"+str(i) if len(str(i))==1 else i}' for i in range(4,20)]
+    print(list_date)
+    count = 1
+    for date_ in list_date:
+        # if count==1:
+        #     conn.command(f"delete from AmazonReport.SD_advertiser_sdAdvertisedProductV3 where date='{date_}'")
+        #     conn.command(f"delete from AmazonReport.SD_asin_sdPurchasedProductV3 where date='{date_}'")
+        #conn.command(f"delete from AmazonReport.SB_sbPurchasedProduct_asinV3 where date='{date_}'")
+        print(date_)
+        print("=======================================")
+        print(date_.replace("-",""))
+        try:
+            amz_report(conn,AWS_CREDENTIALS,para={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        except:
+            time.sleep(15)
+            amz_report(conn,AWS_CREDENTIALS,para={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+        time.sleep(10)
+        count+=1
 
     conn.close()
 

sync_amz_data/public/sp_api_client.py

@@ -1027,7 +1027,7 @@ class SpApiRequest:
         aws_credentials = {
             'refresh_token': refresh_token,
             'lwa_app_id': 'amzn1.application-oa2-client.1f9d3d4747e14b22b4b598e54e6b922e',  # 卖家中心里面开发者资料LWA凭证
-            'lwa_client_secret': 'amzn1.oa2-cs.v1.3af0f5649f5b8e151cd5bd25c10f2bf3113172485cd6ffc52ccc6a5e8512b490',
+            'lwa_client_secret': 'amzn1.oa2-cs.v1.13ebfa8ea7723b478215a2b61f4dbf5ca6f6927fa097c2dd0941a66d510aca7f',
             'aws_access_key': 'AKIARBAGHTGOZC7544GN',
             'aws_secret_key': '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',
             'role_arn': 'arn:aws:iam::070880041373:role/Amazon_SP_API_ROLE'
@@ -1271,7 +1271,7 @@ class SpApiRequest:
             aws_credentials = {
                 'refresh_token': refresh_token,
                 'lwa_app_id': 'amzn1.application-oa2-client.1f9d3d4747e14b22b4b598e54e6b922e',  # 卖家中心里面开发者资料LWA凭证
-                'lwa_client_secret': 'amzn1.oa2-cs.v1.3af0f5649f5b8e151cd5bd25c10f2bf3113172485cd6ffc52ccc6a5e8512b490',
+                'lwa_client_secret': 'amzn1.oa2-cs.v1.13ebfa8ea7723b478215a2b61f4dbf5ca6f6927fa097c2dd0941a66d510aca7f',
                 'aws_access_key': 'AKIARBAGHTGOZC7544GN',
                 'aws_secret_key': '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',
                 'role_arn': 'arn:aws:iam::070880041373:role/Amazon_SP_API_ROLE'
@@ -1330,5 +1330,5 @@ if __name__ == '__main__':
     # print(rel)
     # SpApiRequest.get_allShops("GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT")
     # pass
-    # SpApiRequest.listing_infoTable()
-    SpApiRequest.Goods_drop_duplicates()
+    SpApiRequest.listing_infoTable()
+    # SpApiRequest.Goods_drop_duplicates()