Merge branch 'wengao' of ASJ_ADS/sync_amz_data into master

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbgroup.py

@@ -27,6 +27,18 @@ def request(url_path: str, method: str = "GET", head: dict = None, params: dict
         raise Exception(resp.text)
     return resp.json()
 
+def convert_row(row, klist, tlist, alist):
+    adgroupid = row['adGroupId']
+    if adgroupid in klist:
+        expressions = "keyword"
+    elif adgroupid in tlist:
+        expressions = "target"
+    elif adgroupid in alist:
+        expressions = "target"
+    else:
+        expressions = "notmatch"
+    return expressions
+
 
 class SbGroup:
     def __init__(self, profile_id, campaignId: list = None):
@@ -34,6 +46,9 @@ class SbGroup:
         self.portfolioId = campaignId
         self.re_url_path = "api/ad_manage/profiles/"
         self.upcreate_url_path = "api/ad_manage/sbgroups/updata/"
+        self.kid_url_path = "api/ad_manage/sbkeywordsadgroupId/"
+        self.tid_url_path = "api/ad_manage/sbtargetsadgroupId/"
+        self.aid_url_path = "api/ad_manage/sbadsadgroupId/"
         self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
         self.refresh_token = self.get_refresh_token()
         self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
@@ -57,6 +72,60 @@ class SbGroup:
             out = None
         return out
 
+    def get_keyword_adgroupid_list(self):
+        heads = self.heads
+        url_path = self.kid_url_path
+        data = []
+        page = 1
+        params = {'limit': 999, 'page': page}
+        tem = request(url_path=url_path, head=heads, params=params)
+        data.extend(tem.get('data'))
+        while tem.get('is_next') is True:
+            page += 1
+            params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
+            tem = request(url_path=url_path, head=heads, params=params)
+            data.extend(tem.get('data'))
+        df = pd.json_normalize(data)
+        df.drop_duplicates(inplace=True)
+        kglist = list(df.adGroupId)
+        return kglist
+
+    def get_target_adgroupid_list(self):
+        heads = self.heads
+        url_path = self.tid_url_path
+        data = []
+        page = 1
+        params = {'limit': 999, 'page': page}
+        tem = request(url_path=url_path, head=heads, params=params)
+        data.extend(tem.get('data'))
+        while tem.get('is_next') is True:
+            page += 1
+            params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
+            tem = request(url_path=url_path, head=heads, params=params)
+            data.extend(tem.get('data'))
+        df = pd.json_normalize(data)
+        df.drop_duplicates(inplace=True)
+        tglist = list(df.adGroupId)
+        return tglist
+
+    def get_ads_adgroupid_list(self):
+        heads = self.heads
+        url_path = self.aid_url_path
+        data = []
+        page = 1
+        params = {'limit': 999, 'page': page}
+        tem = request(url_path=url_path, head=heads, params=params)
+        data.extend(tem.get('data'))
+        while tem.get('is_next') is True:
+            page += 1
+            params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
+            tem = request(url_path=url_path, head=heads, params=params)
+            data.extend(tem.get('data'))
+        df = pd.json_normalize(data)
+        df.drop_duplicates(inplace=True)
+        alist = list(df.adGroupId)
+        return alist
+
     def get_spgroup_data(self):
         tem = SBClient(**self.AWS_CREDENTIALS)
         list_group = tem.iter_adGroups(**{"includeExtendedDataFields": True,
@@ -69,6 +138,9 @@ class SbGroup:
 
     def dataconvert(self):
         df = self.get_spgroup_data()
+        tem_t = self.get_target_adgroupid_list()
+        tem_k = self.get_keyword_adgroupid_list()
+        tem_a = self.get_ads_adgroupid_list()
         df['extendedData.creationDate'] = pd.to_datetime(
             df['extendedData.creationDate'], unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
         df['extendedData.lastUpdateDate'] = pd.to_datetime(
@@ -87,6 +159,7 @@ class SbGroup:
             'extendedData_servingStatus': 'servingStatus',
             'extendedData_servingStatusDetails': 'servingStatusDetails'
         }, inplace=True)
+        tem['targetingType'] = tem.apply(convert_row, klist=tem_k, tlist=tem_t, alist=tem_a, axis=1)
         json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
         return json_data
 

sync_amz_data/tasks/datainsert/alldata_insert.py

@@ -55,6 +55,12 @@ spko = spk.updata_create()
 print("SpKeyword", spko)
 protime(start_time)
 
+start_time = time.time()
+spt = SpTargets(profile_id="3006125408623189")
+spto = spt.updata_create()
+print("SpTargets", spto)
+protime(start_time)
+
 start_time = time.time()
 spa = SpAds(profile_id="3006125408623189")
 spao = spa.updata_create()
@@ -73,12 +79,6 @@ sptbo = sptb.updata_create()
 print("SpTargetsBidRecommendations", sptbo)
 protime(start_time)
 
-start_time = time.time()
-spt = SpTargets(profile_id="3006125408623189")
-spto = spt.updata_create()
-print("SpTargets", spto)
-protime(start_time)
-
 start_time = time.time()
 sptbv2 = SpTargetsBidRecommendationsV2(profile_id="3006125408623189")
 sptbv2o = sptbv2.updata_create()