Merge branch 'master' into wengao

start_sync_amz.py

@@ -2,12 +2,15 @@ from sync_amz_data.tasks.account import AccountTask
 from sync_amz_data.settings import LOG_CONF
 import logging.config
 logging.config.dictConfig(LOG_CONF)
-from sync_amz_data.DataTransform import Data_ETL
+from sync_amz_data.public.amz_ad_client import shop_infos
 from sync_amz_data.DataTransform.Data_ETL import Common_ETLMethod,SP_ETL,SB_ETL,SD_ETL
 from apscheduler.schedulers.blocking import BlockingScheduler
 
 
 def amz_report(AWS_CREDENTIALS,para=None):
+    refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
+    AWS_CREDENTIALS['refresh_token'] = refresh_token
+
     conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
     sb_report = SB_ETL(**AWS_CREDENTIALS)
     sb_report.reportV3_purchasedAsinRecord_ETL(conn,params=para)
@@ -76,6 +79,8 @@ if __name__ == '__main__':
         'lwa_client_secret': 'cbf0514186db4df91e04a8905f0a91b605eae4201254ced879d8bb90df4b474d',
         'profile_id': "3006125408623189"
     }
+    refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
+    AWS_CREDENTIALS['refresh_token'] = refresh_token
 
     timezone_ = Common_ETLMethod(**AWS_CREDENTIALS).timeZone()
     print(timezone_)

start_sync_amz_RightNowRun.py

@@ -2,12 +2,15 @@ from sync_amz_data.tasks.account import AccountTask
 from sync_amz_data.settings import LOG_CONF
 import logging.config
 logging.config.dictConfig(LOG_CONF)
-from sync_amz_data.DataTransform import Data_ETL
-from sync_amz_data.DataTransform.Data_ETL import Common_ETLMethod,SP_ETL,SB_ETL,SD_ETL
+import time
+import requests
 
+from sync_amz_data.public.amz_ad_client import shop_infos
+from sync_amz_data.DataTransform.Data_ETL import Common_ETLMethod,SP_ETL,SB_ETL,SD_ETL
 
-def amz_report(AWS_CREDENTIALS,para=None):
-    conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+def amz_report(conn,AWS_CREDENTIALS,para=None):
+    refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
+    AWS_CREDENTIALS['refresh_token'] = refresh_token
     sb_report = SB_ETL(**AWS_CREDENTIALS)
     sb_report.reportV3_purchasedAsinRecord_ETL(conn,params=para)
     sb_report.reportV2_campaignsRecord_ETL(conn,params=para)
@@ -24,9 +27,11 @@ def amz_report(AWS_CREDENTIALS,para=None):
     sb_report.reportV2_searchtermsVideo_ETL(conn,params=para)
     sb_report.reportV2_targetsRecord_ETL(conn,params=para)
     sb_report.reportV2_targetsVideo_ETL(conn,params=para)
-    conn.close()
+    # time.sleep(3)
+    # conn.close()
+    # time.sleep(3)
 
-    conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    # conn = SP_ETL(**AWS_CREDENTIALS).clickhouse_connect()
     sp_report = SP_ETL(**AWS_CREDENTIALS)
     sp_report.reportV3_campaign_spCampaignsETL(conn,params=para)
     sp_report.reportV3_adGroup_spCampaignsETL(conn,params=para)
@@ -35,9 +40,10 @@ def amz_report(AWS_CREDENTIALS,para=None):
     sp_report.reportV3_searchTerm_spSearchTermETL(conn,params=para)
     sp_report.reportV3_advertiser_spAdvertisedProductETL(conn,params=para)
     sp_report.reportV3_asin_spPurchasedProductETL(conn,params=para)
-    conn.close()
-
-    conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
+    # time.sleep(3)
+    # conn.close()
+    # time.sleep(3)
+    # conn = SD_ETL(**AWS_CREDENTIALS).clickhouse_connect()
     sd_report = SD_ETL(**AWS_CREDENTIALS)
 
     sd_report.reportV3_campaign_sdCampaigns_ETL(conn,params=para)
@@ -65,7 +71,6 @@ def amz_report(AWS_CREDENTIALS,para=None):
     # sd_report.reportV2_adGroups_matchedTarget_t3_ETL(conn,params=para)
     # sd_report.reportV2_targets_matchedTarget_t2_ETL(conn,params=para)
     # sd_report.reportV2_targets_matchedTarget_t3_ETL(conn,params=para)
-    conn.close()
 
 if __name__ == '__main__':
     # AccountTask("3006125408623189").do({"record": "portfolios"})
@@ -75,15 +80,20 @@ if __name__ == '__main__':
         'lwa_client_secret': 'cbf0514186db4df91e04a8905f0a91b605eae4201254ced879d8bb90df4b474d',
         'profile_id': "3006125408623189"
     }
-
+    conn = SB_ETL(**AWS_CREDENTIALS).clickhouse_connect()
     timezone_ = Common_ETLMethod(**AWS_CREDENTIALS).timeZone()
     print(timezone_)
-    amz_report(AWS_CREDENTIALS=AWS_CREDENTIALS)
-
 
-    # list_date = ['2023-11-12','2023-11-13']
-    # # # list_date = [f'2023-10-{i}' for i in range(24,23,-1)]
+    refresh_token = shop_infos(AWS_CREDENTIALS['profile_id'])['refresh_token']
+    AWS_CREDENTIALS['refresh_token'] = refresh_token
+    amz_report(conn, AWS_CREDENTIALS=AWS_CREDENTIALS)
+    # list_date = ['2023-11-17',"2023-11-18",]
+    # list_date = [f'2023-11-{"0"+str(i) if len(str(i))==1 else i}' for i in range(13,16)]
+    # print(list_date)
     # for date_ in list_date:
     #     print(date_)
     #     print(date_.replace("-",""))
-    #     amz_report(AWS_CREDENTIALS,para={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+    #     amz_report(conn,AWS_CREDENTIALS,para={"startDate":date_,"endDate":date_,"date":date_.replace("-","")})
+
+
+    conn.close()

sync_amz_data/DataTransform/Data_ETL.py

@@ -3,7 +3,7 @@ from sync_amz_data.public.amz_ad_client import SPClient, Account, SBClient, SDCl
 import pandas as pd
 import numpy as np
 from dateutil.parser import parse
-
+import time
 pd.set_option('display.max_columns', None)
 import warnings
 
@@ -117,8 +117,26 @@ class Common_ETLMethod(BaseClient):
         # df_report[df_report.select_dtypes('O').columns] = df_report[df_report.select_dtypes('O').columns].astype('string')
         toFloat = [i for i in columns if 'sales' in i.lower() or 'percent' in i.lower() or 'video' in i.lower()]
         if len(toFloat) > 0:
-            df_report[toFloat] = df_report[toFloat].applymap(lambda x: np.nan if pd.isna(x) or x == '' else float(x))
-
+            df_report[toFloat] = df_report[toFloat].applymap(lambda x: 0.0 if pd.isna(x) or x == '' else float(x))
+
+        df_report[df_report.select_dtypes(["int"]).columns] = df_report[df_report.select_dtypes(["int"]).columns].fillna(0)
+        df_report[df_report.select_dtypes(["float"]).columns] = df_report[df_report.select_dtypes(["float"]).columns].fillna(0.0)
+        if "campaignRuleBasedBudget" in df_report.columns:
+            df_report["campaignRuleBasedBudget"] = df_report["campaignRuleBasedBudget"].fillna(0.0)
+        if "campaignRuleBasedBudgetAmount" in df_report.columns:
+            df_report["campaignRuleBasedBudgetAmount"] = df_report["campaignRuleBasedBudgetAmount"].fillna(0.0)
+        if "eCPAddToCart" in df_report.columns:
+            df_report["eCPAddToCart"] = df_report["eCPAddToCart"].fillna(0.0)
+        if "eCPBrandSearch" in df_report.columns:
+            df_report["eCPBrandSearch"] =df_report["eCPBrandSearch"].fillna(0.0)
+        if "viewClickThroughRate" in df_report.columns:
+            df_report["viewClickThroughRate"] = df_report["viewClickThroughRate"].fillna(0.0)
+        if "searchTermImpressionRank" in df_report.columns:
+            df_report["searchTermImpressionRank"] = df_report["searchTermImpressionRank"].fillna(0)
+        if "searchTermImpressionShare" in df_report.columns:
+            df_report["searchTermImpressionShare"] = df_report["searchTermImpressionShare"].fillna(0.0)
+        if "impressionsFrequencyAverage" in df_report.columns:
+            df_report["impressionsFrequencyAverage"] = df_report["impressionsFrequencyAverage"].fillna(0.0)
         df_report.fillna(np.nan, inplace=True)
         # print(df_report[columns].info())
         # df_report.to_excel("df.xlsx")
@@ -236,6 +254,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SP_spCampaigns_campaignV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spCampaigns_campaignV3")
 
         df_rel =  df_report[params['columns']]
@@ -271,6 +290,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SP_spCampaigns_adGroupV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spCampaigns_adGroupV3")
 
         df_rel =  df_report[params['columns']]
@@ -308,6 +328,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SP_spCampaigns_placementV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spCampaigns_placementV3")
 
 
@@ -346,6 +367,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SP_spTargeting_targetingV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spTargeting_targetingV3")
 
 
@@ -384,6 +406,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SP_spSearchTerm_searchTermV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spSearchTerm_searchTermV3")
 
 
@@ -419,6 +442,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
         # print(df_report.info())
         conn.insert_df("AmazonReport.SP_spAdvertisedProduct_advertiserV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spAdvertisedProduct_advertiserV3")
 
 
@@ -452,6 +476,7 @@ class SP_ETL(SPClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, params['columns'], timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SP_spPurchasedProduct_asinV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SP_spPurchasedProduct_asinV3")
 
 
@@ -533,6 +558,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         # print(df_report)
         # conn.insert_df("AmazonReport.SB_sbPurchasedProduct_asinV3", df_report[params['columns']])
+        # time.sleep(0.05)
         # print("插入完成SB_sbPurchasedProduct_asinV3")
 
 
@@ -563,6 +589,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SB_sbPurchasedProduct_asinV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SB_sbPurchasedProduct_asinV3")
 
 
@@ -596,6 +623,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SB_campaignsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_campaignsV2")
 
 
@@ -630,6 +658,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SB_campaignsVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_campaignsVideoV2")
         # print(df_report[metric].info())
 
@@ -660,6 +689,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['placement'])
 
         conn.insert_df("AmazonReport.SB_campaignsPlacementV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_campaignsPlacementV2")
         # print(df_report[metric].info())
 
@@ -692,6 +722,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_campaignsPlacementVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_campaignsPlacementVideoV2")
 
 
@@ -723,6 +754,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_adGroupsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_adGroupsV2")
 
 
@@ -755,6 +787,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_adGroupsVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_adGroupsVideoV2")
 
         df_rel =  df_report[metric]
@@ -785,6 +818,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_targetsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_targetsV2")
 
 
@@ -818,6 +852,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_targetsVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_targetsVideoV2")
 
         df_rel =  df_report[metric]
@@ -849,6 +884,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_keywordsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_keywordsV2")
 
         df_rel =  df_report[metric]
@@ -881,6 +917,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_keywordsVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_keywordsVideoV2")
 
         df_rel =  df_report[metric]
@@ -907,6 +944,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_keywordsQueryV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_keywordsQueryV2")
 
         df_rel =  df_report[metric]
@@ -936,6 +974,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_keywordsQueryVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_keywordsQueryVideoV2")
 
         df_rel =  df_report[metric]
@@ -965,6 +1004,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=[])
 
         conn.insert_df("AmazonReport.SB_adsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_adsV2")
 
         df_rel =  df_report[metric]
@@ -998,6 +1038,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SB_adsVideoV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SB_adsVideoV2")
 
         df_rel =  df_report[metric]
@@ -1068,6 +1109,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report.columns)
         conn.insert_df("AmazonReport.SD_sdCampaigns_campaignV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_sdCampaigns_campaignV3")
 
 
@@ -1102,6 +1144,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report.columns)
         conn.insert_df("AmazonReport.SD_sdCampaigns_campaignMatchedTargetV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_sdCampaigns_campaignMatchedTargetV3")
 
 
@@ -1137,6 +1180,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SD_sdAdGroup_adGroupV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_sdAdGroup_adGroupV3")
 
 
@@ -1172,6 +1216,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SD_sdAdGroup_adGroupMatchedTargetV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_sdAdGroup_adGroupMatchedTargetV3")
 
 
@@ -1211,6 +1256,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SD_targeting_sdTargetingV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_targeting_sdTargetingV3")
 
 
@@ -1251,6 +1297,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SD_targeting_sdTargetingMatchedTargetV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_targeting_sdTargetingMatchedTargetV3")
 
 
@@ -1288,6 +1335,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SD_advertiser_sdAdvertisedProductV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_advertiser_sdAdvertisedProductV3")
 
 
@@ -1317,6 +1365,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
 
         # print(df_report)
         conn.insert_df("AmazonReport.SD_asin_sdPurchasedProductV3", df_report[params['columns']])
+        time.sleep(0.05)
         print("插入完成SD_asin_sdPurchasedProductV3")
 
 
@@ -1357,6 +1406,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
 
         conn.insert_df("AmazonReport.SD_campaignsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_campaignsV2—")
 
         df_rel =  df_report[metric]
@@ -1380,6 +1430,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
 
         conn.insert_df("AmazonReport.SD_campaignsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_campaignsV2")
 
         df_rel =  df_report[metric]
@@ -1416,6 +1467,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
 
         conn.insert_df("AmazonReport.SD_adGroupsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_adGroupsV2")
 
         df_rel =  df_report[metric]
@@ -1437,6 +1489,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SD_adGroupsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_adGroupsV2")
 
         df_rel =  df_report[metric]
@@ -1473,6 +1526,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
 
         conn.insert_df("AmazonReport.SD_adsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_adsV2")
 
         df_rel =  df_report[metric]
@@ -1495,6 +1549,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SD_adsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_adsV2")
 
         df_rel =  df_report[metric]
@@ -1531,6 +1586,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
         # print(df_report[metric].info())
         conn.insert_df("AmazonReport.SD_targetsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_targetsV2")
 
         df_rel =  df_report[metric]
@@ -1553,6 +1609,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         # print(df_report.info())
 
         conn.insert_df("AmazonReport.SD_targetsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_targetsV2")
 
         df_rel =  df_report[metric]
@@ -1586,6 +1643,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
         # print(df_report.info())
         conn.insert_df("AmazonReport.SD_asinsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_asinsV2")
 
         df_rel =  df_report[metric]
@@ -1607,6 +1665,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['tactic'])
 
         conn.insert_df("AmazonReport.SD_asinsV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_asinsV2")
 
         df_rel =  df_report[metric]
@@ -1646,6 +1705,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
 
         conn.insert_df("AmazonReport.SD_campaignsMatchedTargetV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_campaignsMatchedTargetV2")
 
         df_rel =  df_report[metric]
@@ -1668,6 +1728,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
 
         conn.insert_df("AmazonReport.SD_campaignsMatchedTargetV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_campaignsMatchedTargetV2")
 
         df_rel =  df_report[metric]
@@ -1704,6 +1765,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
 
         conn.insert_df("AmazonReport.SD_adGroupsMatchedTargetV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_adGroupsMatchedTargetV2")
 
         df_rel =  df_report[metric]
@@ -1724,6 +1786,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report['tactic'] = 'Audiences targeting'  # {"T00020":"Contextual targeting","T00030":"Audiences targeting"}
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
         conn.insert_df("AmazonReport.SD_adGroupsMatchedTargetV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_adGroupsMatchedTargetV2")
 
         df_rel =  df_report[metric]
@@ -1758,6 +1821,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
 
         conn.insert_df("AmazonReport.SD_targetsMatchedTargetV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_targetsMatchedTargetV2")
 
         df_rel =  df_report[metric]
@@ -1778,6 +1842,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_report = self.type_trans(df_report, metric, timeZone_, extra_columns=['matchedTarget', 'tactic'])
 
         conn.insert_df("AmazonReport.SD_targetsMatchedTargetV2", df_report[metric])
+        time.sleep(0.05)
         print("插入完成SD_targetsMatchedTargetV2")
 
         df_rel =  df_report[metric]

sync_amz_data/public/__init__.py

@@ -1,3 +1,4 @@
-from .amz_ad_client import SPClient, SBClient, SDClient, BaseClient, AccountClient
+from .amz_ad_client import SPClient, SBClient, SDClient, BaseClient, AccountClient,shop_infos
+
 
 from .asj_client import asj_api

sync_amz_data/public/sp_api_client.py

@@ -0,0 +1,491 @@
+import clickhouse_connect
+import time
+
+import numpy as np
+from pymysql import Timestamp
+from sp_api.util import throttle_retry, load_all_pages
+from sp_api.api import Orders,ListingsItems,Inventories,Reports
+from sp_api.base import Marketplaces,ReportType,ProcessingStatus
+import pandas as pd
+import gzip
+from io import BytesIO,StringIO
+from datetime import datetime, timedelta,timezone
+import pytz
+import time
+from sync_amz_data.public.amz_ad_client import shop_infos
+from dateutil.parser import parse
+import pymysql
+
+
+
+class SpApiRequest:
+    def __init__(self, credentials, marketplace,profile_id):
+        self.credentials = credentials
+        self.marketplace = marketplace
+        self.shopInfo = shop_infos(profile_id)
+        self.timezone = self.shopInfo['time_zone']
+
+    def mysql_connect(self):
+        conn = pymysql.connect(user="huangyifan",
+        password="123456",
+        host="192.168.1.18",
+        database="amz_sp_api",
+        port=3306)
+        return conn
+
+    def timeToLocalTime(self,para_time=None):
+        if para_time is not None:
+            if ":" not in para_time:
+                para_time = para_time+' 0:0:0'
+            report_USTime = datetime.strptime(para_time,"%Y-%m-%d %H:%M:%S")
+        else:
+            report_USTime = datetime.now(tz=pytz.timezone(self.timezone)) + timedelta(days=-1)
+        USTime = pytz.timezone(self.timezone)
+        ustime_end = USTime.localize(datetime(report_USTime.year,report_USTime.month,report_USTime.day,23,59,59,999999))
+        ustime_start = USTime.localize(datetime(report_USTime.year,report_USTime.month,report_USTime.day,0,0,0))
+        print(self.timezone+": ","\n",ustime_start,"-",ustime_end)
+        # utctime = pytz.timezone("UTC")
+        timezoneutc = pytz.timezone('UTC')
+        startTimeutc = ustime_start.astimezone(timezoneutc)
+        endTimeutc = ustime_end.astimezone(timezoneutc)
+        formatTime = "%Y-%m-%dT%H:%M:%S+00:00"
+        startTimeutc = startTimeutc.strftime(formatTime)
+        endTimeutc = endTimeutc.strftime(formatTime)
+        print("UTC: ","\n",startTimeutc,"-",endTimeutc)
+        return startTimeutc,endTimeutc
+
+    def timeDeal(self,orgTime):
+        orgTime = parse(orgTime)
+        timezone = pytz.timezone(self.timezone)
+        shopTime = orgTime.astimezone(timezone)
+        shopTime_datetime = datetime(shopTime.year,shopTime.month,shopTime.day,shopTime.hour,shopTime.minute,shopTime.second)
+        return shopTime_datetime
+
+
+    def nowTime(self):
+        computerTimenow = datetime.now()
+        localTimeNow = datetime.now(tz=pytz.timezone(self.timezone))
+        utctimeNow = datetime.utcnow()
+        return  {"computerTimenow":computerTimenow,"localTimeNow":localTimeNow,"utctimeNow":utctimeNow}
+
+    def create_report(self,**kwargs):
+        reportType = kwargs['reportType']
+        reportOptions =kwargs.get("reportOptions")
+
+        dataStartTime = datetime.now(tz=pytz.timezone(self.timezone)).strftime("%Y-%m-%dT%H:%M:%S") if kwargs.get("dataStartTime") is None else kwargs.get("dataStartTime")+"T00:00:00"
+        dataEndTime = datetime.now(tz=pytz.timezone(self.timezone)).strftime("%Y-%m-%dT%H:%M:%S") if kwargs.get("dataEndTime") is None else kwargs.get("dataEndTime")+"T23:59:59"
+
+        report = Reports(credentials=self.credentials, marketplace=self.marketplace)
+        rel = report.create_report(
+                        reportType=reportType,marketplaceIds=[self.marketplace.marketplace_id],reportOptions=reportOptions,
+                        dataStartTime=dataStartTime,dataEndTime=dataEndTime
+                        )
+        reportId = rel.payload.get("reportId")
+        print(reportId)
+        return reportId
+
+    def decompression(self,reportId):
+        report = Reports(credentials=self.credentials, marketplace=self.marketplace)
+        while True:
+            reportId_info = report.get_report(reportId=reportId)
+            # print(reportId_info.payload)
+            print("please wait...")
+            if reportId_info.payload.get("processingStatus")==ProcessingStatus.DONE:
+                reportDocumentId = reportId_info.payload.get("reportDocumentId")
+                rp_table = report.get_report_document(reportDocumentId=reportDocumentId,download=False)
+                print(rp_table)
+                if rp_table.payload.get('compressionAlgorithm') is not None:
+                    df = pd.read_table(filepath_or_buffer=rp_table.payload['url'],compression={"method":'gzip'},encoding='iso-8859-1')
+                    return df
+                else:
+                    df = pd.read_table(rp_table.payload.get("url"),encoding='iso-8859-1')
+                    return df
+            elif reportId_info.payload.get("processingStatus") in [ProcessingStatus.CANCELLED,ProcessingStatus.FATAL]:
+                print("取消或失败")
+                break
+            time.sleep(15)
+            print("please wait...")
+
+    def data_deal(self,decom_df):
+        decom_df['mainImageUrl'] = decom_df['seller-sku'].map(lambda x: self.get_mainImage_url(x))
+        url_columns = [i for i in decom_df.columns if "url" in i.lower()]
+        if len(url_columns) > 0:
+            decom_df[url_columns] = decom_df[url_columns].astype("string")
+        asin_columns = [i for i in decom_df.columns if 'asin' in i.lower()]
+        if len(asin_columns) > 0:
+            decom_df[asin_columns] = decom_df[asin_columns].astype("string")
+        if 'pending-quantity' in decom_df.columns:
+            decom_df['pending-quantity'] = decom_df['pending-quantity'].map(
+                lambda x: 0 if pd.isna(x) or np.isinf(x) else x).astype("int32")
+        deletecolumns = [i for i in decom_df.columns if 'zshop' in i.lower()]
+        decom_df.drop(columns=deletecolumns, inplace=True)
+        if 'quantity' in decom_df.columns:
+            decom_df['quantity'] = decom_df['quantity'].map(lambda x: 0 if pd.isna(x) or np.isinf(x) else x).astype(
+                "int32")
+        decom_df['opendate_date'] = decom_df['open-date'].str.split(' ', expand=False).map(lambda x: x[0]).astype(
+            'datetime64[ns]')
+        if 'add-delete' in decom_df.columns:
+            decom_df['add-delete'] = decom_df['add-delete'].astype('string', errors='ignore')
+        if 'will-ship-internationally' in decom_df.columns:
+            decom_df['will-ship-internationally'] = decom_df['will-ship-internationally'].astype('string',errors='ignore')
+        if 'expedited-shipping' in decom_df.columns:
+            decom_df['expedited-shipping'] = decom_df['expedited-shipping'].astype('string',errors='ignore')
+        decom_df['updateTime'] = datetime.now(tz=pytz.timezone(self.timezone))
+        decom_df['timezone'] = self.timezone
+        decom_df['item-description'] = decom_df['item-description'].str.slice(0,500)
+        decom_df[decom_df.select_dtypes(float).columns] = decom_df[decom_df.select_dtypes(float).columns].fillna(0.0)
+        decom_df[decom_df.select_dtypes(int).columns] = decom_df[decom_df.select_dtypes(int).columns].fillna(0)
+        decom_df[decom_df.select_dtypes(datetime).columns] = decom_df[decom_df.select_dtypes(datetime).columns].astype(
+            'string')
+        decom_df.fillna('', inplace=True)
+        return decom_df
+
+    def GET_MERCHANT_LISTINGS_ALL_DATA(self,limit=None):
+        para = {"reportType":ReportType.GET_MERCHANT_LISTINGS_ALL_DATA}
+        reportid = self.create_report(**para)
+        decom_df = self.decompression(reportid)
+        print("连接数据库")
+        conn = self.mysql_connect()
+        cursor = conn.cursor()
+        bondary_date = (datetime.today() + timedelta(days=-28)).strftime("%Y-%m-%d")
+        cursor.execute(f"""select * from amz_sp_api.productInfo where (mainImageUrl is not null and mainImageUrl not in ('', ' ')) and 
+                        (`seller-sku` not in ('',' ') and `seller-sku` is not null) and 
+                        `updateTime`>='{bondary_date}'""") #`seller-sku`,`updateTime`,`mainImageUrl`
+        col = [i[0] for i in cursor.description]
+        query_rel = cursor.fetchall()
+        print(query_rel[0])
+        if len(query_rel)!=0:
+            df = pd.DataFrame(query_rel,columns=col)
+            listingid = df['listing-id'].to_numpy().tolist()
+            decom_df = decom_df.query("`listing-id` not in @listingid")
+            # print(f"delete * from amz_sp_api.productInfo where `listing-id` not in {tuple(listingid)}")
+            cursor.execute(f"delete from amz_sp_api.productInfo where `listing-id` not in {tuple(listingid)}")
+            conn.commit()
+
+        if len(decom_df)==0:
+            return "Done"
+
+        if limit != None:
+            decom_df = decom_df.iloc[:limit,:]
+        print("getting mainImageInfo...")
+        rowcount = 0
+        while rowcount < len(decom_df):
+            df_insert = decom_df.copy()
+            df_insert = df_insert.iloc[rowcount:rowcount + 200, :]
+
+            df_insert = self.data_deal(df_insert)
+            list_df = df_insert.to_numpy().tolist()
+
+            # print(list(conn.query("select * from amz_sp_api.orderReport")))
+            sql = f"""
+                        insert into amz_sp_api.productInfo
+                        values (%s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s)
+                    """
+            # print(sql)
+            try:
+                conn.begin()
+                cursor.executemany(sql, list_df)
+                conn.commit()
+                print("插入中...")
+                rowcount += 200
+            except Exception as e:
+                conn.rollback()
+                print(e)
+        print("全部完成")
+        return decom_df
+
+    def get_mainImage_url(self, sku):
+        listingClient = ListingsItems(credentials=self.credentials, marketplace=self.marketplace)
+        try:
+            r1 = listingClient.get_listings_item(sellerId=self.shopInfo['advertiser_id'], sku=sku)
+            img = r1.payload.get("summaries")[0].get("mainImage")
+            img_url = None if img is None else img.get("link")
+        except Exception as e:
+            print(e)
+            time.sleep(3)
+            r1 = listingClient.get_listings_item(sellerId=self.shopInfo['advertiser_id'], sku=sku)
+            img = r1.payload.get("summaries")[0].get("mainImage")
+            img_url = None if img is None else img.get("link")
+        return img_url
+
+    def GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL(self):
+        timezone_ = pytz.timezone(self.timezone)
+        shopReportday = (datetime.now(tz=timezone_) + timedelta(days=-1)).strftime("%Y-%m-%d")
+        # print(shopReportday)
+        para = {"reportType":ReportType.GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL,"dataStartTime":shopReportday,"dataEndTime":shopReportday,"reportOptions":{"ShowSalesChannel":"true"}}
+        reportid = self.create_report(**para) #{"ShowSalesChannel":"true"}
+        decom_df = self.decompression(reportid)
+        decom_df[["purchase-date","last-updated-date"]] = decom_df[["purchase-date","last-updated-date"]].applymap(lambda x: self.timeDeal(x) if pd.isna(x)==False or x != None else x)
+
+        decom_df[decom_df.select_dtypes(float).columns] = decom_df[decom_df.select_dtypes(float).columns].fillna(0.0)
+        decom_df[decom_df.select_dtypes(int).columns] = decom_df[decom_df.select_dtypes(int).columns].fillna(0)
+        decom_df[decom_df.select_dtypes(datetime).columns] = decom_df[decom_df.select_dtypes(datetime).columns].astype('string')
+        if "purchase-order-number" in decom_df.columns:
+            decom_df['purchase-order-number'] = decom_df['purchase-order-number'].astype("string")
+        decom_df.fillna('',inplace=True)
+        decom_df["ReportDate"] = parse(shopReportday)
+        decom_df['timezone'] = self.timezone
+        list_df = decom_df.to_numpy().tolist()
+        print(list_df[0])
+        # tuple_data = [tuple(i) for i in list_df]
+        conn = self.mysql_connect()
+        cursor = conn.cursor()
+        # print(list(conn.query("select * from amz_sp_api.orderReport")))
+        sql = f"""
+            insert into amz_sp_api.orderReport
+            values (%s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s, %s,%s,%s,%s,%s,%s,%s)
+        """
+        # print(sql)
+        try:
+            conn.begin()
+            cursor.executemany(sql,list_df)
+            conn.commit()
+            print("插入完成")
+        except Exception as e:
+            conn.rollback()
+            print(e)
+
+    @throttle_retry()
+    @load_all_pages()
+    def load_all_orders(self,**kwargs):
+        """
+        a generator function to return all pages, obtained by NextToken
+        """
+        return Orders(credentials=self.credentials, marketplace=Marketplaces.US).get_orders(**kwargs)
+
+    @throttle_retry()
+    @load_all_pages()
+    def load_order_items(self,**kwargs):
+        return Orders(credentials=self.credentials).get_order_items(**kwargs)
+
+    def order_instant(self, sleep_time=0.8):
+        insert_list_orderBasic = []
+        insert_list_orderDetail = []
+        orderId_list = []
+        row = 0
+        dt_ = datetime.utcnow()
+        LastUpdatedAfter = (datetime(dt_.year, dt_.month, dt_.day, dt_.hour, 0, 0, 0) - timedelta(hours=1)).isoformat()
+        LastUpdatedBefore = (datetime(dt_.year, dt_.month, dt_.day, dt_.hour, 59, 59, 59) - timedelta(hours=1)).isoformat()
+        print(LastUpdatedAfter, '-', LastUpdatedBefore)
+        for page in self.load_all_orders(LastUpdatedAfter=LastUpdatedAfter, LastUpdatedBefore=LastUpdatedBefore):  #
+            for order in page.payload.get('Orders'):
+                # print(order)
+                AmazonOrderId = str(order.get('AmazonOrderId'))
+                orderId_list.append(AmazonOrderId)
+                try:
+                    items_ = self.load_order_items(order_id=str(order.get('AmazonOrderId')))
+                except:
+                    try:
+                        time.sleep(3)
+                        items_ = self.load_order_items(order_id=str(order.get('AmazonOrderId')))
+                    except:
+                        time.sleep(5)
+                        items_ = self.load_order_items(order_id=str(order.get('AmazonOrderId')))
+                # print(next(items_).payload)
+                items_detail = next(items_).payload["OrderItems"][0]
+                TaxCollection_Model, TaxCollection_ResponsibleParty = (None, None) if items_detail.get(
+                    "TaxCollection") == None else (items_detail.get("TaxCollection").get("Model"),
+                                                   items_detail.get("TaxCollection").get("ResponsibleParty"))
+                ProductInfo_NumberOfItems = None if items_detail.get("ProductInfo") == None else int(
+                    items_detail.get("ProductInfo").get("NumberOfItems"))
+                BuyerInfo = None if items_detail.get("BuyerInfo") == None else str(items_detail.get("BuyerInfo"))
+                CurrencyCode = None if items_detail.get("ItemPrice") == None else items_detail.get("ItemPrice").get(
+                    "CurrencyCode")
+                ItemTax_Amount = None if items_detail.get("ItemTax") == None else float(
+                    items_detail.get("ItemTax").get("Amount"))
+                QuantityShipped = None if items_detail.get("QuantityShipped") == None else int(
+                    items_detail.get("QuantityShipped"))
+                ItemPrice_Amount = None if items_detail.get("ItemPrice") == None else float(
+                    items_detail.get("ItemPrice").get("Amount"))
+                ASIN = None if items_detail.get("ASIN") == None else items_detail.get("ASIN")
+                SellerSKU = None if items_detail.get("SellerSKU") == None else items_detail.get("SellerSKU")
+                Title = None if items_detail.get("Title") == None else items_detail.get("Title")
+                ShippingTax_Amount = None if items_detail.get("ShippingTax") == None else float(
+                    items_detail.get("ShippingTax").get("Amount"))
+                IsGift = None if items_detail.get("IsGift") == None else (
+                    False if items_detail.get("IsGift") == 'false' else True)
+                PriceDesignation = None if items_detail.get("PriceDesignation") == None else items_detail.get(
+                    "PriceDesignation")
+                ShippingPrice_Amount = None if items_detail.get("ShippingPrice") == None else float(
+                    items_detail.get("ShippingPrice").get("Amount"))
+                ShippingDiscount_Amount = None if items_detail.get("ShippingDiscount") == None else float(
+                    items_detail.get("ShippingDiscount").get("Amount"))
+                ShippingDiscountTax_Amount = None if items_detail.get("ShippingDiscountTax") == None else float(
+                    items_detail.get("ShippingDiscountTax").get("Amount"))
+                IsTransparency = None if items_detail.get("IsTransparency") == None else items_detail.get(
+                    "IsTransparency")
+                QuantityOrdered = None if items_detail.get("QuantityOrdered") == None else int(
+                    items_detail.get("QuantityOrdered"))
+                PromotionDiscountTax_Amount = None if items_detail.get("PromotionDiscountTax") == None else float(
+                    items_detail.get("PromotionDiscountTax").get("Amount"))
+                PromotionDiscount_Amount = None if items_detail.get("PromotionDiscount") == None else float(
+                    items_detail.get("PromotionDiscount").get("Amount"))
+                OrderItemId = None if items_detail.get("OrderItemId") == None else str(items_detail.get("OrderItemId"))
+                temp_inserttime = datetime.now(tz=pytz.timezone(self.timezone)) + timedelta(hours=1)
+                Inserthour_time = datetime(temp_inserttime.year, temp_inserttime.month, temp_inserttime.day,temp_inserttime.hour, 0, 0)
+                insert_list_orderDetail.append(
+                    [AmazonOrderId, TaxCollection_Model, TaxCollection_ResponsibleParty, ProductInfo_NumberOfItems,
+                     BuyerInfo,
+                     CurrencyCode, ItemTax_Amount, QuantityShipped, ItemPrice_Amount, ASIN, SellerSKU, Title,
+                     ShippingTax_Amount, IsGift, PriceDesignation, ShippingPrice_Amount,
+                     ShippingDiscount_Amount, ShippingDiscountTax_Amount, IsTransparency, QuantityOrdered,
+                     PromotionDiscountTax_Amount, PromotionDiscount_Amount, OrderItemId, Inserthour_time,self.timezone])
+
+                BuyerInfo_BuyerEmail = None if order.get('BuyerInfo') == None else order.get('BuyerInfo').get(
+                    'BuyerEmail')
+                AmazonOrderId = AmazonOrderId  # order.get('AmazonOrderId') == None else order.get('AmazonOrderId')
+
+                EarliestDeliveryDate = None if order.get('EarliestDeliveryDate') == None else self.timeDeal(order.get('EarliestDeliveryDate'))
+
+                EarliestShipDate = None if order.get('EarliestShipDate') == None else self.timeDeal(order.get('EarliestShipDate'))
+
+                SalesChannel = None if order.get('SalesChannel') == None else order.get('SalesChannel')
+                AutomatedShippingSettings_HasAutomatedShippingSettings = None if order.get('AutomatedShippingSettings') == None else order.get('AutomatedShippingSettings').get('HasAutomatedShippingSettings')
+                OrderStatus = None if order.get('OrderStatus') == None else order.get('OrderStatus')
+                NumberOfItemsShipped = None if order.get('NumberOfItemsShipped') == None else order.get('NumberOfItemsShipped')
+                OrderType = None if order.get('OrderType') == None else order.get('OrderType')
+                IsPremiumOrder = None if order.get('IsPremiumOrder') == None else order.get('IsPremiumOrder')
+                IsPrime = None if order.get('IsPrime') == None else order.get('IsPrime')
+                FulfillmentChannel = None if order.get('FulfillmentChannel') == None else order.get('FulfillmentChannel')
+                NumberOfItemsUnshipped = None if order.get('NumberOfItemsUnshipped') == None else order.get('NumberOfItemsUnshipped')
+                HasRegulatedItems = None if order.get('HasRegulatedItems') == None else order.get('HasRegulatedItems')
+                IsReplacementOrder = None if order.get('IsReplacementOrder') == None else eval(order.get('IsReplacementOrder').capitalize())
+                IsSoldByAB = None if order.get('IsSoldByAB') == None else order.get('IsSoldByAB')
+                LatestShipDate = None if order.get('LatestShipDate') == None else self.timeDeal(order.get('LatestShipDate'))
+                ShipServiceLevel = None if order.get('ShipServiceLevel') == None else order.get('ShipServiceLevel')
+                DefaultShipFromLocationAddress_StateOrRegion = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('StateOrRegion')
+                DefaultShipFromLocationAddress_AddressLine1 = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('AddressLine1')
+                DefaultShipFromLocationAddress_Phone = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('Phone')
+                DefaultShipFromLocationAddress_PostalCode = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('PostalCode')
+                DefaultShipFromLocationAddress_City = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('City')
+                DefaultShipFromLocationAddress_CountryCode = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('CountryCode')
+                DefaultShipFromLocationAddress_Name = None if order.get('DefaultShipFromLocationAddress') == None else order.get('DefaultShipFromLocationAddress').get('Name')
+                IsISPU = None if order.get('IsISPU') == None else order.get('IsISPU')
+                MarketplaceId = None if order.get('MarketplaceId') == None else order.get('MarketplaceId')
+                LatestDeliveryDate = None if order.get('LatestDeliveryDate') == None else self.timeDeal(order.get('LatestDeliveryDate'))
+                PurchaseDate = None if order.get('PurchaseDate') == None else self.timeDeal(order.get('PurchaseDate'))
+                ShippingAddress_StateOrRegion = None if order.get('ShippingAddress') == None else order.get('ShippingAddress').get('StateOrRegion')
+                ShippingAddress_PostalCode = None if order.get('ShippingAddress') == None else order.get('ShippingAddress').get('PostalCode')
+                ShippingAddress_City = None if order.get('ShippingAddress') == None else order.get('ShippingAddress').get('City')
+                ShippingAddress_CountryCode = None if order.get('ShippingAddress') == None else order.get('ShippingAddress').get('CountryCode')
+                IsAccessPointOrder = None if order.get('IsAccessPointOrder') == None else order.get('IsAccessPointOrder')
+                PaymentMethod = None if order.get('PaymentMethod') == None else order.get('PaymentMethod')
+                IsBusinessOrder = None if order.get('IsBusinessOrder') == None else order.get('IsBusinessOrder')
+                OrderTotal_CurrencyCode = None if order.get('OrderTotal') == None else order.get('OrderTotal').get('CurrencyCode')
+                OrderTotal_Amount = None if order.get('OrderTotal') == None else float(order.get('OrderTotal').get('Amount'))
+                PaymentMethodDetails = None if order.get('PaymentMethodDetails') == None else order.get('PaymentMethodDetails')[0]
+                IsGlobalExpressEnabled = None if order.get('IsGlobalExpressEnabled') == None else order.get('IsGlobalExpressEnabled')
+                LastUpdateDate = None if order.get('LastUpdateDate') == None else self.timeDeal(order.get('LastUpdateDate'))
+                ShipmentServiceLevelCategory = None if order.get('ShipmentServiceLevelCategory') == None else order.get('ShipmentServiceLevelCategory')
+                SellerOrderId = None if order.get('SellerOrderId') == None else order.get('SellerOrderId')
+                temp_inserttime =  datetime.now(tz=pytz.timezone(self.timezone))+timedelta(hours=1)
+                Inserthour_time = datetime(temp_inserttime.year,temp_inserttime.month,temp_inserttime.day,temp_inserttime.hour,0,0)
+
+                insert_list_orderBasic.append(
+                    [BuyerInfo_BuyerEmail, AmazonOrderId, EarliestDeliveryDate, EarliestShipDate, SalesChannel,
+                     AutomatedShippingSettings_HasAutomatedShippingSettings, OrderStatus,
+                     NumberOfItemsShipped, OrderType, IsPremiumOrder, IsPrime, FulfillmentChannel,
+                     NumberOfItemsUnshipped, HasRegulatedItems, IsReplacementOrder, IsSoldByAB, LatestShipDate,
+                     ShipServiceLevel, DefaultShipFromLocationAddress_StateOrRegion,
+                     DefaultShipFromLocationAddress_AddressLine1, DefaultShipFromLocationAddress_Phone,
+                     DefaultShipFromLocationAddress_PostalCode,
+                     DefaultShipFromLocationAddress_City, DefaultShipFromLocationAddress_CountryCode,
+                     DefaultShipFromLocationAddress_Name, IsISPU, MarketplaceId, LatestDeliveryDate, PurchaseDate,
+                     ShippingAddress_StateOrRegion, ShippingAddress_PostalCode, ShippingAddress_City,
+                     ShippingAddress_CountryCode, IsAccessPointOrder, PaymentMethod, IsBusinessOrder,
+                     OrderTotal_CurrencyCode,
+                     OrderTotal_Amount, PaymentMethodDetails, IsGlobalExpressEnabled, LastUpdateDate,
+                     ShipmentServiceLevelCategory, SellerOrderId, Inserthour_time,self.timezone])
+
+                time.sleep(sleep_time)
+                row += 1
+                if row % 10 == 0:
+                    print(f"receiving...received {row} rows")
+        print(insert_list_orderBasic)
+        print(insert_list_orderDetail)
+if __name__ == '__main__':
+    aws_credentials = {
+        'refresh_token': 'Atzr|IwEBIMxC7d17ZYBTcNe-zfnbk-TEC-40uIlSRmGAH_sfNozcA7RdSt4iBdUorC2GC_uoUmIY4oGhHGT621el1my0YaABEvuGn4eIe1EFxrHYLM1fljnQxHQjgGKwTb48cMdHNxOiMd8_CbvLYL_NT3E_zTPKCSQjqa8zxTkddBjL-5stlTRzhVHmB2Ox5-6XgvpSnoBtlZqkI96rYmKi63f0NTQ2e9IV3cNqXli8X9_DYGgZlpl60qE56A-ZNy4otv4myR5kqY2bTll0c-ynLtqI5ukDwPwO369b_Ie4kniRd7Or-Ip6jPyTIdfghLV4AVrxbvE',
+        'lwa_app_id': 'amzn1.application-oa2-client.1f9d3d4747e14b22b4b598e54e6b922e',  # 卖家中心里面开发者资料LWA凭证
+        'lwa_client_secret': 'amzn1.oa2-cs.v1.3af0f5649f5b8e151cd5bd25c10f2bf3113172485cd6ffc52ccc6a5e8512b490',
+        'aws_access_key': 'AKIARBAGHTGOZC7544GN',
+        'aws_secret_key': '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',
+        'role_arn': 'arn:aws:iam::070880041373:role/Amazon_SP_API_ROLE'
+    }
+    sp_ = SpApiRequest(aws_credentials,Marketplaces.US,'3006125408623189')
+    sp_.GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL()
+    """
+    create database amz_sp_api;
+    """
+    """
+            create table amz_sp_api.productInfo
+            (
+            `item-name`	VARCHAR(300),
+            `item-description`	VARCHAR(1000),
+            `listing-id`	VARCHAR(50),
+            `seller-sku`	VARCHAR(50),
+            `price`	FLOAT,
+            `quantity`	INT,
+            `open-date`	VARCHAR(70),
+            `image-url`	VARCHAR(300),
+            `item-is-marketplace`	VARCHAR(50),
+            `product-id-type`	INT,
+            `item-note`	VARCHAR(300),
+            `item-condition`	INT,
+            `asin1`	VARCHAR(50),
+            `asin2`	VARCHAR(50),
+            `asin3`	VARCHAR(50),
+            `will-ship-internationally`	VARCHAR(50),
+            `expedited-shipping`	VARCHAR(50),
+            `product-id`	VARCHAR(50),
+            `bid-for-featured-placement`	FLOAT,
+            `add-delete`	VARCHAR(50),
+            `pending-quantity`	INT,
+            `fulfillment-channel`	VARCHAR(50),
+            `merchant-shipping-group`	VARCHAR(50),
+            `status`	VARCHAR(50),
+            `mainImageUrl`	VARCHAR(300),
+            `opendate_date`	Date,
+            `updateTime`	Date,
+            `timezone`	VARCHAR(30)
+            )
+    """
+    """
+    create table amz_sp_api.orderReport
+    (`amazon-order-id` VARCHAR(40),
+    `merchant-order-id` VARCHAR(40),
+    `purchase-date` DATETIME,
+    `last-updated-date` DATETIME,
+    `order-status` VARCHAR(40),
+    `fulfillment-channel` VARCHAR(40),
+    `sales-channel` VARCHAR(40),
+    `order-channel` VARCHAR(40),
+    `ship-service-level` VARCHAR(40),
+    `product-name` VARCHAR(250),
+    `sku` VARCHAR(50),
+    `asin` VARCHAR(40),
+    `item-status` VARCHAR(40),
+    `quantity` INT,
+    `currency` VARCHAR(40),
+    `item-price` FLOAT,
+    `item-tax` FLOAT,
+    `shipping-price` FLOAT,
+    `shipping-tax` FLOAT,
+    `gift-wrap-price` FLOAT,
+    `gift-wrap-tax` FLOAT,
+    `item-promotion-discount` FLOAT,
+    `ship-promotion-discount` FLOAT,
+    `ship-city` VARCHAR(40),
+    `ship-state` VARCHAR(40),
+    `ship-postal-code` VARCHAR(40),
+    `ship-country` VARCHAR(40),
+    `promotion-ids` VARCHAR(50),
+    `cpf` VARCHAR(40),
+    `is-business-order` BOOL,
+    `purchase-order-number` VARCHAR(50),
+    `price-designation` VARCHAR(40),
+    `signature-confirmation-recommended` BOOL,
+    `ReportDate` DATE not null,
+    `timezone` VARCHAR(20) not null
+    );
+    """