modify files

start_sync_amz.py

@@ -2,7 +2,7 @@ from sync_amz_data.tasks.account import AccountTask
 from sync_amz_data.settings import LOG_CONF
 import logging.config
 logging.config.dictConfig(LOG_CONF)
-from sync_amz_data.DataTransform import Data_ETL
+from sync_amz_data.public.amz_ad_client import shop_infos
 from sync_amz_data.DataTransform.Data_ETL import Common_ETLMethod,SP_ETL,SB_ETL,SD_ETL
 from apscheduler.schedulers.blocking import BlockingScheduler
 
@@ -76,6 +76,8 @@ if __name__ == '__main__':
         'lwa_client_secret': 'cbf0514186db4df91e04a8905f0a91b605eae4201254ced879d8bb90df4b474d',
         'profile_id': "3006125408623189"
     }
+    refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
+    AWS_CREDENTIALS['refresh_token'] = refresh_token
 
     timezone_ = Common_ETLMethod(**AWS_CREDENTIALS).timeZone()
     print(timezone_)

start_sync_amz_RightNowRun.py

@@ -4,12 +4,12 @@ import logging.config
 logging.config.dictConfig(LOG_CONF)
 import time
 
-from sync_amz_data.DataTransform import Data_ETL
+from sync_amz_data.public.amz_ad_client import shop_infos
 from sync_amz_data.DataTransform.Data_ETL import Common_ETLMethod,SP_ETL,SB_ETL,SD_ETL
 
 
-def amz_report(AWS_CREDENTIALS,para=None):
-    conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+def amz_report(conn,AWS_CREDENTIALS,para=None):
+
     sb_report = SB_ETL(**AWS_CREDENTIALS)
     sb_report.reportV3_purchasedAsinRecord_ETL(conn,params=para)
     sb_report.reportV2_campaignsRecord_ETL(conn,params=para)
@@ -26,11 +26,11 @@ def amz_report(AWS_CREDENTIALS,para=None):
     sb_report.reportV2_searchtermsVideo_ETL(conn,params=para)
     sb_report.reportV2_targetsRecord_ETL(conn,params=para)
     sb_report.reportV2_targetsVideo_ETL(conn,params=para)
-    time.sleep(3)
-    conn.close()
-    time.sleep(3)
+    # time.sleep(3)
+    # conn.close()
+    # time.sleep(3)
 
-    conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    # conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
     sp_report = SP_ETL(**AWS_CREDENTIALS)
     sp_report.reportV3_campaign_spCampaignsETL(conn,params=para)
     sp_report.reportV3_adGroup_spCampaignsETL(conn,params=para)
@@ -39,10 +39,10 @@ def amz_report(AWS_CREDENTIALS,para=None):
     sp_report.reportV3_searchTerm_spSearchTermETL(conn,params=para)
     sp_report.reportV3_advertiser_spAdvertisedProductETL(conn,params=para)
     sp_report.reportV3_asin_spPurchasedProductETL(conn,params=para)
-    time.sleep(3)
-    conn.close()
-    time.sleep(3)
-    conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    # time.sleep(3)
+    # conn.close()
+    # time.sleep(3)
+    # conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
     sd_report = SD_ETL(**AWS_CREDENTIALS)
 
     sd_report.reportV3_campaign_sdCampaigns_ETL(conn,params=para)
@@ -70,7 +70,7 @@ def amz_report(AWS_CREDENTIALS,para=None):
     # sd_report.reportV2_adGroups_matchedTarget_t3_ETL(conn,params=para)
     # sd_report.reportV2_targets_matchedTarget_t2_ETL(conn,params=para)
     # sd_report.reportV2_targets_matchedTarget_t3_ETL(conn,params=para)
-    conn.close()
+
 
 if __name__ == '__main__':
     # AccountTask("3006125408623189").do({"record": "portfolios"})
@@ -80,15 +80,20 @@ if __name__ == '__main__':
         'lwa_client_secret': 'cbf0514186db4df91e04a8905f0a91b605eae4201254ced879d8bb90df4b474d',
         'profile_id': "3006125408623189"
     }
-
+    conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
     timezone_ = Common_ETLMethod(**AWS_CREDENTIALS).timeZone()
     print(timezone_)
-    amz_report(AWS_CREDENTIALS=AWS_CREDENTIALS)
-
+    amz_report(conn,AWS_CREDENTIALS=AWS_CREDENTIALS)
+    refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
+    AWS_CREDENTIALS['refresh_token'] = refresh_token
 
-    # list_date = ['2023-11-02',]
-    # # # list_date = [f'2023-10-{i}' for i in range(24,23,-1)]
+    # list_date = ['2023-11-05',"2023-11-06","2023-11-07"]
+    # list_date = [f'2023-11-{"0"+str(i) if len(str(i))==1 else i}' for i in range(13,16)]
+    # print(list_date)
     # for date_ in list_date:
     #     print(date_)
     #     print(date_.replace("-",""))
-    #     amz_report(AWS_CREDENTIALS,para={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+    #     amz_report(conn,AWS_CREDENTIALS,para={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+
+
+    conn.close()

sync_amz_data/DataTransform/Data_ETL.py

@@ -3,7 +3,7 @@ from sync_amz_data.public.amz_ad_client import SPClient, Account, SBClient, SDCl
 import pandas as pd
 import numpy as np
 from dateutil.parser import parse
-
+import time
 pd.set_option('display.max_columns', None)
 import warnings
 
@@ -254,6 +254,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SP_spCampaigns_campaignV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spCampaigns_campaignV3")
 
         df_rel =  df_report[params['columns']]
@@ -289,6 +290,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SP_spCampaigns_adGroupV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spCampaigns_adGroupV3")
 
         df_rel =  df_report[params['columns']]
@@ -326,6 +328,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SP_spCampaigns_placementV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spCampaigns_placementV3")
 
 
@@ -364,6 +367,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SP_spTargeting_targetingV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spTargeting_targetingV3")
 
 
@@ -402,6 +406,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SP_spSearchTerm_searchTermV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spSearchTerm_searchTermV3")
 
 
@@ -437,6 +442,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
         # print(df_report.info())
         conn.insert_df("AmazonReport.SP_spAdvertisedProduct_advertiserV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spAdvertisedProduct_advertiserV3")
 
 
@@ -470,6 +476,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SP_spPurchasedProduct_asinV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spPurchasedProduct_asinV3")
 
 
@@ -551,6 +558,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         # print(df_report)
         # conn.insert_df("AmazonReport.SB_sbPurchasedProduct_asinV3", df_report[params['columns']])
+        # time.sleep(0.05)
         # print("插入完成SB_sbPurchasedProduct_asinV3")
 
 
@@ -581,6 +589,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SB_sbPurchasedProduct_asinV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SB_sbPurchasedProduct_asinV3")
 
 
@@ -614,6 +623,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SB_campaignsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_campaignsV2")
 
 
@@ -648,6 +658,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SB_campaignsVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_campaignsVideoV2")
         # print(df_report[metric].info())
 
@@ -678,6 +689,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['placement'])
 
         conn.insert_df("AmazonReport.SB_campaignsPlacementV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_campaignsPlacementV2")
         # print(df_report[metric].info())
 
@@ -710,6 +722,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_campaignsPlacementVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_campaignsPlacementVideoV2")
 
 
@@ -741,6 +754,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_adGroupsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_adGroupsV2")
 
 
@@ -773,6 +787,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_adGroupsVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_adGroupsVideoV2")
 
         df_rel =  df_report[metric]
@@ -803,6 +818,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_targetsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_targetsV2")
 
 
@@ -836,6 +852,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_targetsVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_targetsVideoV2")
 
         df_rel =  df_report[metric]
@@ -867,6 +884,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_keywordsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_keywordsV2")
 
         df_rel =  df_report[metric]
@@ -899,6 +917,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_keywordsVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_keywordsVideoV2")
 
         df_rel =  df_report[metric]
@@ -925,6 +944,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_keywordsQueryV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_keywordsQueryV2")
 
         df_rel =  df_report[metric]
@@ -954,6 +974,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_keywordsQueryVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_keywordsQueryVideoV2")
 
         df_rel =  df_report[metric]
@@ -983,6 +1004,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SB_adsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_adsV2")
 
         df_rel =  df_report[metric]
@@ -1016,6 +1038,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_adsVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_adsVideoV2")
 
         df_rel =  df_report[metric]
@@ -1086,6 +1109,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report.columns)
         conn.insert_df("AmazonReport.SD_sdCampaigns_campaignV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_sdCampaigns_campaignV3")
 
 
@@ -1120,6 +1144,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report.columns)
         conn.insert_df("AmazonReport.SD_sdCampaigns_campaignMatchedTargetV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_sdCampaigns_campaignMatchedTargetV3")
 
 
@@ -1155,6 +1180,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SD_sdAdGroup_adGroupV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_sdAdGroup_adGroupV3")
 
 
@@ -1190,6 +1216,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SD_sdAdGroup_adGroupMatchedTargetV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_sdAdGroup_adGroupMatchedTargetV3")
 
 
@@ -1229,6 +1256,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SD_targeting_sdTargetingV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_targeting_sdTargetingV3")
 
 
@@ -1269,6 +1297,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SD_targeting_sdTargetingMatchedTargetV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_targeting_sdTargetingMatchedTargetV3")
 
 
@@ -1306,6 +1335,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SD_advertiser_sdAdvertisedProductV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_advertiser_sdAdvertisedProductV3")
 
 
@@ -1335,6 +1365,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SD_asin_sdPurchasedProductV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_asin_sdPurchasedProductV3")
 
 
@@ -1375,6 +1406,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
 
         conn.insert_df("AmazonReport.SD_campaignsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_campaignsV2—")
 
         df_rel =  df_report[metric]
@@ -1398,6 +1430,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
 
         conn.insert_df("AmazonReport.SD_campaignsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_campaignsV2")
 
         df_rel =  df_report[metric]
@@ -1434,6 +1467,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
 
         conn.insert_df("AmazonReport.SD_adGroupsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_adGroupsV2")
 
         df_rel =  df_report[metric]
@@ -1455,6 +1489,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SD_adGroupsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_adGroupsV2")
 
         df_rel =  df_report[metric]
@@ -1491,6 +1526,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
 
         conn.insert_df("AmazonReport.SD_adsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_adsV2")
 
         df_rel =  df_report[metric]
@@ -1513,6 +1549,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SD_adsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_adsV2")
 
         df_rel =  df_report[metric]
@@ -1549,6 +1586,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
         # print(df_report[metric].info())
         conn.insert_df("AmazonReport.SD_targetsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_targetsV2")
 
         df_rel =  df_report[metric]
@@ -1571,6 +1609,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SD_targetsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_targetsV2")
 
         df_rel =  df_report[metric]
@@ -1604,6 +1643,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
         # print(df_report.info())
         conn.insert_df("AmazonReport.SD_asinsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_asinsV2")
 
         df_rel =  df_report[metric]
@@ -1625,6 +1665,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
 
         conn.insert_df("AmazonReport.SD_asinsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_asinsV2")
 
         df_rel =  df_report[metric]
@@ -1664,6 +1705,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
 
         conn.insert_df("AmazonReport.SD_campaignsMatchedTargetV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_campaignsMatchedTargetV2")
 
         df_rel =  df_report[metric]
@@ -1686,6 +1728,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
 
         conn.insert_df("AmazonReport.SD_campaignsMatchedTargetV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_campaignsMatchedTargetV2")
 
         df_rel =  df_report[metric]
@@ -1722,6 +1765,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
 
         conn.insert_df("AmazonReport.SD_adGroupsMatchedTargetV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_adGroupsMatchedTargetV2")
 
         df_rel =  df_report[metric]
@@ -1742,6 +1786,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report['tactic'] = 'Audiences targeting'  # {"T00020":"Contextual targeting","T00030":"Audiences targeting"}
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
         conn.insert_df("AmazonReport.SD_adGroupsMatchedTargetV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_adGroupsMatchedTargetV2")
 
         df_rel =  df_report[metric]
@@ -1776,6 +1821,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
 
         conn.insert_df("AmazonReport.SD_targetsMatchedTargetV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_targetsMatchedTargetV2")
 
         df_rel =  df_report[metric]
@@ -1796,6 +1842,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
 
         conn.insert_df("AmazonReport.SD_targetsMatchedTargetV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_targetsMatchedTargetV2")
 
         df_rel =  df_report[metric]