Merge branch 'wengao'

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sp_targetsbid_recommendations.py

@@ -5,6 +5,7 @@ from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
 from sync_amz_data.tasks.datainsert.wg import LADS
+import time
 
 
 class RateLimitError(Exception):
@@ -117,10 +118,22 @@ class SpTargetsBidRecommendations:
         return json_data
 
     def updata_create(self):
-        body = self.get_sptargetsbidrecommendation_data()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        # body = self.get_sptargetsbidrecommendation_data()
+        # heads = self.heads
+        # url_path = self.upcreate_url_path
+        # tem = request(url_path=url_path, head=heads, body=body, method="POST")      
+        my_list = self.get_sptargetsbidrecommendation_data()
+        chunk_size = 500
+        chunked_lists = [my_list[i:i + chunk_size] for i in range(0, len(my_list), chunk_size)]
+        temlist = []
+        for body in chunked_lists:
+            heads = self.heads
+            url_path = self.upcreate_url_path
+            tem = request(url_path=url_path, head=heads, body=body, method="POST")
+            temlist.append(tem)
+            time.sleep(1)
+        return temlist
+        
         return tem
 
 

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spads.py

@@ -65,9 +65,9 @@ class SpAds:
 
     def dataconvert(self):
         df = self.get_spads_data()
-        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
+        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime'], format='ISO8601').dt.strftime(
             '%Y-%m-%d %H:%M:%S')
-        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
+        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime'], format='ISO8601').dt.strftime(
             '%Y-%m-%d %H:%M:%S')
         col = ['adId', 'campaignId', 'customText', 'asin', 'state', 'sku', 'adGroupId', 'extendedData.creationDateTime',
                'extendedData.lastUpdateDateTime', 'extendedData.servingStatus', 'extendedData.servingStatusDetails']

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spcampaign.py

@@ -90,9 +90,9 @@ class SpCampaign:
     def dataconvert(self):
         df = self.get_spcampaign_data()
         df['profile_id'] = self.profile_id
-        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
+        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime'], format='ISO8601').dt.strftime(
             '%Y-%m-%d %H:%M:%S')
-        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
+        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime'], format='ISO8601').dt.strftime(
             '%Y-%m-%d %H:%M:%S')
         col = ['campaignId', 'endDate', 'name', 'startDate', 'state', 'targetingType',
                'budget.budget', 'budget.budgetType', 'budget.effectiveBudget',

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spgroup.py

@@ -116,9 +116,9 @@ class SpGroup:
         tem_t = self.get_target_adgroupid_list()
         tem_k = self.get_keyword_adgroupid_list()
         df = self.get_spgroup_data()
-        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
+        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime'], format='ISO8601').dt.strftime(
             '%Y-%m-%d %H:%M:%S')
-        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
+        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime'], format='ISO8601').dt.strftime(
             '%Y-%m-%d %H:%M:%S')
         col = ['adGroupId', 'campaignId', 'defaultBid', 'name', 'state', 'extendedData.creationDateTime',
                'extendedData.lastUpdateDateTime', 'extendedData.servingStatus', 'extendedData.servingStatusDetails']

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spkeyword.py

@@ -67,9 +67,9 @@ class SpKeyword:
     def dataconvert(self):
         df = self.get_spkeywords_data()
         df['profile_id'] = self.profile_id
-        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
+        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime'], format='ISO8601').dt.strftime(
             '%Y-%m-%d %H:%M:%S')
-        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
+        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime'], format='ISO8601').dt.strftime(
             '%Y-%m-%d %H:%M:%S')
         col = ['keywordId', 'nativeLanguageKeyword', 'nativeLanguageLocale', 'campaignId', 'matchType', 'state', 'bid',
                'adGroupId', 'keywordText', 'extendedData.creationDateTime',

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spnegativekeyword.py

@@ -65,9 +65,9 @@ class SpNegativeKeyword:
     def dataconvert(self):
         df = self.get_spnegativekeywords_data()
         df['profile'] = self.profile_id
-        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
+        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime'], format='ISO8601').dt.strftime(
             '%Y-%m-%d %H:%M:%S')
-        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
+        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime'], format='ISO8601').dt.strftime(
             '%Y-%m-%d %H:%M:%S')
         col = ['keywordId', 'campaignId', 'matchType', 'state', 'adGroupId', 'keywordText',
                'extendedData.creationDateTime', 'extendedData.lastUpdateDateTime',

sync_amz_data/tasks/datainsert/mysql_datainsert_portfolios.py

@@ -74,6 +74,8 @@ class Portfolios:
         old = pd.DataFrame(data=[], columns=col)
         tem = pd.concat([old, df])
         tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem['budget_startDate'] = pd.to_datetime(tem['budget_startDate'], format='%Y%m%d').dt.strftime('%Y-%m-%d')
+        tem['budget_endDate'] = pd.to_datetime(tem['budget_endDate'], format='%Y%m%d').dt.strftime('%Y-%m-%d')
         json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
         return json_data
 
@@ -88,4 +90,4 @@ if __name__ == '__main__':
     a = Portfolios("3006125408623189")
     out = a.updata_create()
     # out = a.dataconvert()
-    print(out)
+    print(out)

sync_amz_data/tasks/datainsert/wg.py

@@ -1,2 +1,2 @@
-# LADS = "http://192.168.1.36:8001/"
+# LADS = "http://192.168.1.19:8001/"
 LADS = "https://ads.vzzon.com"