Merge branch 'wengao' of ASJ_ADS/sync_amz_data into master

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbcampaign_v4.py

@@ -0,0 +1,118 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SBClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SbCampaign:
+    def __init__(self, profile_id, portfolioId: list = None):
+        self.profile_id = profile_id
+        self.portfolioId = portfolioId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/sbcampaigns/list/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_sbcampaign_data(self):
+        tem = SBClient(**self.AWS_CREDENTIALS)
+        if self.portfolioId is None:
+            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True})
+        else:
+            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True,
+                                                   "portfolioIdFilter": {"include": self.portfolioId}
+                                                   })
+        df_campaigns = pd.json_normalize(list(list_campaigns))
+        return df_campaigns
+
+    def dataconvert(self):
+        df = self.get_sbcampaign_data()
+        df['profile_id'] = self.profile_id
+        df['extendedData.creationDate'] = pd.to_datetime(df['extendedData.creationDate'], unit='ms').dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        df['extendedData.lastUpdateDate'] = pd.to_datetime(df['extendedData.lastUpdateDate'], unit='ms').dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        col = ['budgetType', 'ruleBasedBudget.isProcessing', 'ruleBasedBudget.applicableRuleName',
+               'ruleBasedBudget.value', 'ruleBasedBudget.applicableRuleId', 'brandEntityId',
+               'isMultiAdGroupsEnabled', 'goal', 'bidding.bidOptimization', 'bidding.bidAdjustmentsByShopperSegment',
+               'bidding.bidAdjustmentsByPlacement', 'bidding.bidOptimizationStrategy', 'endDate',
+               'campaignId', 'productLocation', 'portfolioId', 'costType', 'smartDefault', 'name', 'state',
+               'startDate', 'budget', 'extendedData.servingStatus', 'extendedData.lastUpdateDate',
+               'extendedData.creationDate', 'extendedData.servingStatusDetails']
+        old = pd.DataFrame(data=[], columns=col)
+        tem = pd.concat([old, df]).reset_index()
+        tem.drop(columns='index', inplace=True)
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem.rename(columns={
+            'name': 'campaignName',
+            'portfolioId': 'portfolio',
+            'ruleBasedBudget_isProcessing': 'rbb_isProcessing',
+            'ruleBasedBudget_applicableRuleName': 'rbb_appRuleName',
+            'ruleBasedBudget_value': 'rbb_value',
+            'ruleBasedBudget_applicableRuleId': 'rbb_appRuleId',
+            'bidding_bidOptimization': 'bidOptimization',
+            'bidding_bidAdjustmentsByShopperSegment': 'bidShopperSegment',
+            'bidding_bidAdjustmentsByPlacement': 'bidPlacement',
+            'bidding_bidOptimizationStrategy': 'bidOptimizationStrategy',
+            'extendedData_lastUpdateDate': 'lastUpdateDate',
+            'extendedData_servingStatus': 'servingStatus',
+            'extendedData_servingStatusDetails': 'servingStatusDetails',
+            'extendedData_creationDate': 'creationDate'}, inplace=True)
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SbCampaign(profile_id="3006125408623189")
+    out = a.updata_create()
+    # out = a.dataconvert()
+    print(out)

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbkeyword_v3.py

@@ -0,0 +1,97 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SBClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SpKeyword:
+    def __init__(self, profile_id, campaignId: list = None):
+        self.profile_id = profile_id
+        self.portfolioId = campaignId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/sbkeywords/list/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_sbkeywords_data(self):
+        tem = SBClient(**self.AWS_CREDENTIALS)
+        _ = tem.iter_keywords()
+        list_sbkeywords = [row for i in _ for row in i]
+        df_sbkeywords = pd.json_normalize(list(list_sbkeywords))
+        return df_sbkeywords
+
+    def dataconvert(self):
+        df = self.get_sbkeywords_data()
+        col = ['keywordId', 'adGroupId', 'campaignId', 'keywordText',
+               'nativeLanguageKeyword', 'matchType', 'state', 'bid']
+        old = pd.DataFrame(data=[], columns=col)
+        for i in col:
+            if i not in df.columns:
+                df['{}'.format(i)] = None
+        df_new = df[col].copy()
+        tem = pd.concat([old, df_new]).reset_index()
+        tem.drop(columns='index', inplace=True)
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem.rename(columns={
+            'adGroupId': 'adGroup',
+            'campaignId': 'campaign',
+        }, inplace=True)
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SpKeyword(profile_id="3006125408623189")
+    out = a.updata_create()
+    # out = a.dataconvert()
+    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spkeyword.py

@@ -32,7 +32,7 @@ class SpKeyword:
         self.profile_id = profile_id
         self.portfolioId = campaignId
         self.re_url_path = "api/ad_manage/profiles/"
-        self.upcreate_url_path = "api/keywords/list/"
+        self.upcreate_url_path = "api/spkeywords/list/"
         self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
         self.refresh_token = self.get_refresh_token()
         self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
@@ -64,6 +64,7 @@ class SpKeyword:
 
     def dataconvert(self):
         df = self.get_spkeywords_data()
+        df['profile_id'] = self.profile_id
         df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
             '%Y-%m-%d %H:%M:%S')
         df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(