Merge branch 'wengao' of ASJ_ADS/sync_amz_data into master

sync_amz_data/public/amz_ad_client.py

@@ -631,7 +631,7 @@ class SBClient(BaseClient):
 
     def get_keyword(self,keywordid):
         url_path = f'/sb/keywords/{keywordid}'
-        return self._request(url_path,method="GET")
+        return self._request(url_path, method="GET")
 
     def iter_keywords(self,**param):
         if "startIndex" not in param:
@@ -660,6 +660,24 @@ class SBClient(BaseClient):
                 break
             body["nextToken"] = info["nextToken"]
 
+    def get_targets_groupid(self, adgroupIds: list):
+        url_path = "/sb/targets/list"
+        headers = {
+            "Content-Type": "application/json"
+        }
+        body = {"maxResults": 5000, "filters": [{"filterType": "AD_GROUP_ID", "values": adgroupIds}]}
+        return self._request(url_path, method="POST", body=body, headers=headers)
+
+    def iter_targets_groupid(self, adgroupIds):
+        for i in range(0, len(adgroupIds), 50):
+            try:
+                info = self.get_keyword_bidrecommendation(adgroupIds[i:i + 50])
+                yield from info['targets']
+            except:
+                # print("空值")
+                return iter([])
+
+
     def get_budget(self, campaignIds: list):
         url_path = "/sb/campaigns/budget/usage"
         body = {"campaignIds": campaignIds}

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbgroup.py

@@ -34,7 +34,7 @@ def convert_row(row, klist, tlist, alist):
     elif adgroupid in tlist:
         expressions = "target"
     elif adgroupid in alist:
-        expressions = "target"
+        expressions = "ads"
     else:
         expressions = "notmatch"
     return expressions

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbtarget_v3.py

@@ -5,6 +5,8 @@ from sync_amz_data.settings import AWS_LWA_CLIENT
 import pandas as pd
 import json
 from sync_amz_data.tasks.datainsert.wg import LADS
+from sync_amz_data.public.amz_ad_client import SBClient
+from typing import Literal
 
 
 class RateLimitError(Exception):
@@ -12,7 +14,7 @@ class RateLimitError(Exception):
         self.retry_after = retry_after
 
 
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD = LADS):
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None, AD=LADS):
     ADS = AD
     resp = requests.session().request(
         method=method,
@@ -32,6 +34,7 @@ class SbTargets:
     def __init__(self, profile_id):
         self.profile_id = profile_id
         self.re_url_path = "api/ad_manage/profiles/"
+        self.gid_url_path = "api/ad_manage/getsbadgroupId/"
         self.upcreate_url_path = "api/ad_manage/sbtargets/updata/"
         self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
         self.refresh_token = self.get_refresh_token()
@@ -56,6 +59,30 @@ class SbTargets:
             out = None
         return out
 
+    def id_type_trans(self, df):
+        id_columns = [i for i in df.columns if 'id' in i.lower() and 'bid' not in i.lower()]
+        df[id_columns] = df[id_columns].fillna(-1).applymap(lambda x: round(int(x), 0)).astype("string")
+        df[id_columns] = df[id_columns].applymap(lambda x: '' if x == '-1' else x)
+        return df
+
+    @classmethod
+    def get_keyOvalue(cls, expressions, result: Literal['value', 'type']):
+        if expressions not in [None, pd.NA, pd.NaT]:
+            if len(expressions) > 1:
+                return [i.get(result) for i in expressions]
+            if len(expressions) == 1:
+                sub_ = expressions[0].get(result)
+                if sub_ is None:
+                    return None
+                elif type(sub_) == str:
+                    return sub_
+                elif type(sub_) == list:
+                    return [i.get(result) for i in sub_]
+                else:
+                    return expressions
+        else:
+            return '-'
+
     def get_sbtargets_data(self):
         tem = SB_ETL(**self.AWS_CREDENTIALS)
         df = tem.targets_ETL()
@@ -71,8 +98,51 @@ class SbTargets:
             df['profile'] = self.profile_id
         return df
 
+    def get_adgroup_data(self):
+        heads = self.heads
+        url_path = self.gid_url_path
+        data = []
+        page = 1
+        params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
+        tem = request(url_path=url_path, head=heads, params=params)
+        data.extend(tem.get('data'))
+        while tem.get('is_next') is True:
+            page += 1
+            params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
+            tem = request(url_path=url_path, head=heads, params=params)
+            data.extend(tem.get('data'))
+        df = pd.json_normalize(data)
+        out_list_id = list(df.adGroupId.values)
+        return out_list_id
+
+    def get_sbtargets_data_new(self):
+        tem = SBClient(**self.AWS_CREDENTIALS)
+        adgroupids = self.get_adgroup_data()
+        _ = tem.get_targets_groupid(adgroupIds=adgroupids)
+        df = _.get('targets')
+        df_targets = pd.json_normalize(df)
+        if len(df_targets) > 0:
+            df_targets = self.id_type_trans(df_targets)
+            df_targets['expressions_type'] = df_targets['expressions'].map(
+                lambda x: self.get_keyOvalue(x, 'type'))
+            df_targets['expressions_value'] = df_targets['expressions'].map(
+                lambda x: self.get_keyOvalue(x, 'value'))
+            df_targets['resolvedExpressions_type'] = df_targets['resolvedExpressions'].map(
+                lambda x: self.get_keyOvalue(x, 'type'))
+            df_targets['resolvedExpressions_value'] = df_targets['resolvedExpressions'].map(
+                lambda x: self.get_keyOvalue(x, 'value'))
+            df_targets.rename(columns={'adGroupId': 'adGroup',
+                                       'campaignId': 'campaign',
+                                       'expressions_type': 'expression_type',
+                                       'expressions_value': 'expression_value',
+                                       'resolvedExpressions_type': 'resolvedExpression_type',
+                                       'resolvedExpressions_value': 'resolvedExpression_value'}, inplace=True)
+            df_targets.drop(columns=['expressions', 'resolvedExpressions'], inplace=True)
+            df_targets['profile'] = self.profile_id
+        return df_targets
+
     def updata_create(self):
-        df_data = self.get_sbtargets_data()
+        df_data = self.get_sbtargets_data_new()
         _ = df_data.to_json(orient='records', date_format='iso')
         body = json.loads(_)
         heads = self.heads
@@ -81,9 +151,8 @@ class SbTargets:
         return tem
 
 
-
 if __name__ == '__main__':
     a = SbTargets(profile_id="3006125408623189")
-    # out = a.get_sbtargets_data()
+    # out = a.get_sbtargets_data_new()
     out = a.updata_create()
     print(out)