Merge branch 'master' into yifan

sync_amz_data/DataTransform/Data_ETL.py

@@ -218,6 +218,20 @@ class SP_ETL(SPClient, Common_ETLMethod):
             lambda x: self.get_keyOvalue(x, 'value'))
         return self.columnsName_modify(df_targets)
 
+    def negative_targets_ETL(self):
+        list_targets = list(self.iter_negativetargeting(**{"includeExtendedDataFields": True}))
+        df_targets = pd.json_normalize(list_targets)
+        df_targets = self.TZ_Deal(df_targets, ["extendedData.creationDateTime", "extendedData.lastUpdateDateTime"])
+        df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(
+            lambda x: self.get_keyOvalue(x, 'type'))
+        df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(
+            lambda x: self.get_keyOvalue(x, 'value'))
+        df_targets['expression_type'] = df_targets['expression'].map(
+            lambda x: self.get_keyOvalue(x, 'type'))
+        df_targets['expression_value'] = df_targets['expression'].map(
+            lambda x: self.get_keyOvalue(x, 'value'))
+        return self.columnsName_modify(df_targets)
+
     def budget_ETL(self, campaign_ids: list):
         list_budget = self.get_budget(campaign_ids=campaign_ids)['success']
         df_budget = pd.json_normalize(list_budget)

sync_amz_data/public/amz_ad_client.py

@@ -289,10 +289,23 @@ class SPClient(BaseClient):
         url_path = "/v2/sp/targets/bidRecommendations"
         body = {
                 "adGroupId": adGroupId,
-                "expressions": [expressions]
+                "expressions": expressions
                 }
         return self._request(url_path,method="POST",body=body)
 
+    def iter_bidrecommendationList(self, adGroupId, expressions):
+        for i in range(0,len(expressions), 10):
+            try:
+                info = self.get_bidrecommendationList(adGroupId, expressions[i:i+10])
+                out = info['recommendations']
+                for i in out:
+                    i['adGroupId'] = adGroupId
+                yield from out
+            except:
+                # print("空值")
+                return iter([])
+
+
     def get_campaignNegativekeyword(self,**body):
         url_path = '/sp/campaignNegativeKeywords/list'
         headers = {

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spcampaign.py

@@ -27,6 +27,26 @@ def request(url_path: str, method: str = "GET", head: dict = None, params: dict
     return resp.json()
 
 
+def convert_row(row):
+    l_t = None
+    l_p = None
+    l_r = None
+    if len(row['dyp']) > 0:
+        for i in row['dyp']:
+            a = json.loads(i)
+            if 'placement' in a:
+                if a['placement'] == "PLACEMENT_TOP":
+                    l_t = a['percentage']
+                if a['placement'] == "PLACEMENT_PRODUCT_PAGE":
+                    l_p = a['percentage']
+                if a['placement'] == "PLACEMENT_REST_OF_SEARCH":
+                    l_r = a['percentage']
+    return l_t, l_p, l_r
+# row['top'],row['product_page'],row['rest_of_search']
+
+def parse_dict_list(lst):
+    return [json.dumps(json_str) for json_str in lst]
+
 class SpCampaign:
     def __init__(self, profile_id, portfolioId: list = None):
         self.profile_id = profile_id
@@ -84,6 +104,10 @@ class SpCampaign:
         tem = pd.concat([old, df]).reset_index()
         tem.drop(columns='index', inplace=True)
         tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem['dyp'] = tem['dynamicBidding_placementBidding']
+        tem.dyp = tem.dyp.apply(parse_dict_list)
+        tem[['top', 'product_page', 'rest_of_search']] = tem.apply(convert_row, axis=1, result_type='expand')
+        tem.drop(columns='dyp', inplace=True)
         tem.rename(columns={
             'name': 'campaignName',
             'portfolioId': 'portfolio',

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spgroup.py

@@ -27,12 +27,25 @@ def request(url_path: str, method: str = "GET", head: dict = None, params: dict
     return resp.json()
 
 
+def convert_row(row, klist, tlist):
+    adgroupid = row['adGroupId']
+    if adgroupid in klist:
+        expressions = "keyword"
+    elif adgroupid in tlist:
+        expressions = "target"
+    else:
+        expressions = "notmatch"
+    return expressions
+
+
 class SpGroup:
     def __init__(self, profile_id, campaignId: list = None):
         self.profile_id = profile_id
         self.portfolioId = campaignId
         self.re_url_path = "api/ad_manage/profiles/"
         self.upcreate_url_path = "api/ad_manage/spgroups/updata/?updata=1"
+        self.kid_url_path = "api/ad_manage/spkeywordsadgroupId/"
+        self.tid_url_path = "api/ad_manage/sptargetsadgroupId/"
         self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
         self.refresh_token = self.get_refresh_token()
         self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
@@ -63,7 +76,45 @@ class SpGroup:
         return df_group
     #----------------------------
 
+    def get_keyword_adgroupid_list(self):
+        heads = self.heads
+        url_path = self.kid_url_path
+        data = []
+        page = 1
+        params = {'limit': 999, 'page': page}
+        tem = request(url_path=url_path, head=heads, params=params)
+        data.extend(tem.get('data'))
+        while tem.get('is_next') is True:
+            page += 1
+            params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
+            tem = request(url_path=url_path, head=heads, params=params)
+            data.extend(tem.get('data'))
+        df = pd.json_normalize(data)
+        df.drop_duplicates(inplace=True)
+        kglist = list(df.adGroupId)
+        return kglist
+
+    def get_target_adgroupid_list(self):
+        heads = self.heads
+        url_path = self.tid_url_path
+        data = []
+        page = 1
+        params = {'limit': 999, 'page': page}
+        tem = request(url_path=url_path, head=heads, params=params)
+        data.extend(tem.get('data'))
+        while tem.get('is_next') is True:
+            page += 1
+            params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
+            tem = request(url_path=url_path, head=heads, params=params)
+            data.extend(tem.get('data'))
+        df = pd.json_normalize(data)
+        df.drop_duplicates(inplace=True)
+        tglist = list(df.adGroupId)
+        return tglist
+
     def dataconvert(self):
+        tem_t = self.get_target_adgroupid_list()
+        tem_k = self.get_keyword_adgroupid_list()
         df = self.get_spgroup_data()
         df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
             '%Y-%m-%d %H:%M:%S')
@@ -83,6 +134,7 @@ class SpGroup:
             'extendedData_servingStatus': 'servingStatus',
             'extendedData_servingStatusDetails': 'servingStatusDetails'
         }, inplace=True)
+        tem['targetingType'] = tem.apply(convert_row, klist=tem_k, tlist=tem_t,axis=1)
         json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
         return json_data
 
@@ -98,6 +150,7 @@ if __name__ == '__main__':
     a = SpGroup(profile_id="3006125408623189")
     out = a.updata_create()
     # out = a.dataconvert()
+    # out = a.get_target_adgroupid_list()
     print(out)
 
 

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spnegativekeyword.py

@@ -0,0 +1,101 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SPClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SpNegativeKeyword:
+    def __init__(self, profile_id, campaignId: list = None):
+        self.profile_id = profile_id
+        self.campaignId = campaignId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/spnegativekeyword/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_spnegativekeywords_data(self):
+        tem = SPClient(**self.AWS_CREDENTIALS)
+        list_spkeywords = tem.iter_negativekeyword(**{"includeExtendedDataFields": True})
+        df_spnegativekeywords = pd.json_normalize(list(list_spkeywords))
+        return df_spnegativekeywords
+
+    def dataconvert(self):
+        df = self.get_spnegativekeywords_data()
+        df['profile'] = self.profile_id
+        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        col = ['keywordId', 'campaignId', 'matchType', 'state', 'adGroupId', 'keywordText',
+               'extendedData.creationDateTime', 'extendedData.lastUpdateDateTime',
+               'extendedData.servingStatus', 'extendedData.servingStatusDetails']
+        old = pd.DataFrame(data=[], columns=col)
+        tem = pd.concat([old, df]).reset_index()
+        tem.drop(columns='index', inplace=True)
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem.rename(columns={
+            'adGroupId': 'adGroup',
+            'campaignId': 'campaign',
+            'extendedData_creationDateTime': 'creationDateTime',
+            'extendedData_lastUpdateDateTime': 'lastUpdateDateTime',
+            'extendedData_servingStatus': 'servingStatus',
+            'extendedData_servingStatusDetails': 'servingStatusDetails'
+        }, inplace=True)
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+if __name__ == '__main__':
+    a = SpNegativeKeyword(profile_id="3006125408623189")
+    # out = a.get_spnegativekeywords_data()
+    out = a.updata_create()
+    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spnegativetarget.py

@@ -0,0 +1,97 @@
+from sync_amz_data.DataTransform.Data_ETL import SP_ETL
+
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SPClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SpNegativeTarget:
+    def __init__(self, profile_id):
+        self.profile_id = profile_id
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/spnegativetarget/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+
+    def get_spnegativetarget_data(self):
+        tem = SP_ETL(**self.AWS_CREDENTIALS)
+        df = tem.negative_targets_ETL()
+        if len(df) > 0:
+            df.rename(columns={'adgroupid': 'adGroup',
+                               'campaignid': 'campaign',
+                               'expressiontype': 'expressionType',
+                               'targetid': 'targetId',
+                               'extendeddata_creationdatetime': 'creationDateTime',
+                               'extendeddata_lastupdatedatetime': 'lastUpdateDateTime',
+                               'extendeddata_servingstatus': 'servingStatus',
+                               'extendeddata_servingstatusdetails': 'servingStatusDetails',
+                               'resolvedexpressions_type': 'resolvedExpression_type',
+                               'resolvedexpressions_value': 'resolvedExpression_value'}, inplace=True)
+            df.drop(columns=['expression', 'resolvedexpression'], inplace=True)
+            df['profile'] = self.profile_id
+            df['creationDateTime'] = pd.to_datetime(df['creationDateTime']).dt.strftime(
+                '%Y-%m-%d %H:%M:%S')
+            df['lastUpdateDateTime'] = pd.to_datetime(df['lastUpdateDateTime']).dt.strftime(
+                '%Y-%m-%d %H:%M:%S')
+        return df
+
+    def updata_create(self):
+        df_data = self.get_spnegativetarget_data()
+        _ = df_data.to_json(orient='records', date_format='iso')
+        body = json.loads(_)
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SpNegativeTarget(profile_id="3006125408623189")
+    # out = a.get_spnegativetarget_data()
+    out = a.updata_create()
+    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sptarget.py

@@ -95,3 +95,4 @@ if __name__ == '__main__':
     # out = a.get_sptargets_data()
     out = a.updata_create()
     print(out)
+

sync_amz_data/tasks/datainsert/alldata_insert.py

@@ -7,6 +7,8 @@ from SP.mysql_datainsert_spads import SpAds
 from SP.mysql_datainsert_sp_targetsbid_recommendations import SpTargetsBidRecommendations
 from SP.mysql_datainsert_sptarget import SpTargets
 from SP.mysql_datainsert_sp_targetsbid_recommendations_v2 import SpTargetsBidRecommendationsV2
+from SP.mysql_datainsert_spnegativekeyword import SpNegativeKeyword
+from SP.mysql_datainsert_spnegativetarget import SpNegativeTarget
 
 
 pf = Portfolios("3006125408623189")
@@ -44,3 +46,13 @@ print("SpTargets", spto)
 sptbv2 = SpTargetsBidRecommendationsV2(profile_id="3006125408623189")
 sptbv2o = sptbv2.updata_create()
 print("SpTargetsBidRecommendationsV2", sptbv2o)
+
+spnt = SpNegativeTarget(profile_id="3006125408623189")
+spnto = spnt.updata_create()
+print('SpNegativeTarget', spnto)
+
+spnk = SpNegativeKeyword(profile_id="3006125408623189")
+spnko = spnk.updata_create()
+print('SpNegativeKeyword', spnko)
+
+