Merge branch 'yifan' of ASJ_ADS/sync_amz_data into master

sync_amz_data/DataTransform/Data_ETL.py

@@ -78,13 +78,15 @@ class Common_ETLMethod(BaseClient):
     def get_keyOvalue(self,expressions, result: Literal['value', 'type']):
         if expressions not in [None,pd.NA,pd.NaT]:
             if len(expressions) > 1:
-                return [i[result] for i in expressions]
+                return [i.get(result) for i in expressions]
             if len(expressions) == 1:
-                sub_ = expressions[0][result]
-                if type(sub_) == str:
+                sub_ = expressions[0].get(result)
+                if sub_ is None:
+                    return None
+                elif type(sub_) == str:
                     return sub_
                 elif type(sub_) == list:
-                    return [i[result] for i in sub_]
+                    return [i.get(result) for i in sub_]
                 else:
                     return expressions
         else:
@@ -167,9 +169,14 @@ class SP_ETL(SPClient, Common_ETLMethod):
         return self.columnsName_modify(df_keywords_SP)
 
     def targets_ETL(self):
-        list_targets = list(self.iter_targets())
+        list_targets = list(self.iter_targets(**{"includeExtendedDataFields": True}))
         df_targets = pd.json_normalize(list_targets)
+        print("--")
         df_targets = self.TZ_Deal(df_targets, ["extendedData.creationDateTime", "extendedData.lastUpdateDateTime"])
+        df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(
+            lambda x: self.get_keyOvalue(x, 'type'))
+        df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(
+            lambda x: self.get_keyOvalue(x, 'value'))
         return self.columnsName_modify(df_targets)
 
     def budget_ETL(self, campaign_ids: list):
@@ -1002,8 +1009,8 @@ class SD_ETL(SDClient, Common_ETLMethod):
         df_targets = pd.json_normalize(list_targets)
         # df_targets = self.expression_split(df_targets, "resolvedExpression")
         df_targets = self.id_type_trans(df_targets)
-        df_targets['resolvedExpressions_type'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'type'))
-        df_targets['resolvedExpressions_value'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'value'))
+        df_targets['resolvedExpressions_type'] = df_targets['resolvedExpression'].map(lambda x:self.get_keyOvalue(x,'type'))
+        df_targets['resolvedExpressions_value'] = df_targets['resolvedExpression'].map(lambda x:self.get_keyOvalue(x,'value'))
         return self.columnsName_modify(df_targets)
 
     def budget_ETL(self, campaignsIds: list):
@@ -1784,9 +1791,12 @@ if __name__ == '__main__':
     conn = Common_ETLMethod(**AWS_CREDENTIALS).clickhouse_connect()
 
     # SD
-    ac_etl = SD_ETL(**AWS_CREDENTIALS)
-    ls = ac_etl.reportV2_campaignsRecord_t2_ETL(conn)
+    ac_etl = SB_ETL(**AWS_CREDENTIALS)
+    # ls = ac_etl.reportV2_campaignsRecord_t2_ETL(conn)
+    ls = ac_etl.targets_ETL()
     print(ls)
+    # ls.to_csv('temp.csv')
+    # print(ls)
     # print(ls.info())
     # print(ls.to_excel('obse11.xlsx'))
     # ac_etl.reportV2_campaignsRecord_t2_ETL(conn)

sync_amz_data/tasks/__init__.py

@@ -1,2 +0,0 @@
-from .sb import SBTask
-

sync_amz_data/tasks/_base.py

@@ -1,65 +0,0 @@
-from functools import lru_cache
-import logging
-from typing import List
-
-from sync_amz_data.public import BaseClient, asj_api
-from sync_amz_data.settings import AWS_LWA_CLIENT, DATA_PATH
-
-logger = logging.getLogger(__name__)
-ASJ_URL_BASE = "http://127.0.0.1:8000/api/ad_manage/"
-
-
-@lru_cache(maxsize=100)
-def query_shop_info(profile_id: str) -> dict:
-    # todo 临时
-    if profile_id == "3006125408623189":
-        return {
-            "profile_id": profile_id,
-            "id": 1,
-            "shop_name": "ZosiDirect",
-            "region": "NA",
-            "access_token": "Atza|IwEBION12cAeJLW1CrhirYbH8ianuTfh1JQtC2nbnZjWcCk_J0v9XVF00Pm7AYzZPHXAuFgdul0vuQt4XUAcwVdzQF9AzDfk5wKAUXe9fuGZhb0nq0mdJxg08u2BR_rUggxKWSd1sg6OW7szIEq8xQzok9hcr-Ai-WupyaA-CCznOt7STmyEZltNsK8VuJb7ySxTlxwf-DbuX2Tn9JdsEta7DwQIsHcYv2QwsfLYGnk2LcLUUXG-6TtwCWFGHMxHfoScOvN92hOHiPl3CsdTs5RmKO5eVFdf0XUu8OU5Z9icnjuP1tYBy7_e9s2oTL9fVLLfH_ATUJplPsfm1MhtW6ioX9IXxfJSmOJ0ntEv45ndb9t-wHE_vLukuy_4jwmy_50NU_TOU_9pbFJLQhVyAB0f4HVcb5fgPn--feAX89ANhRJnIn5zibVk_rY_rte7Xu7JMJNNW41PoCUvfVdxjnorhGTIGh2u2JHQPqdww2xLcZ93SQ",
-            "status": 1,
-            "refresh_token": "Atzr|IwEBIL4ur8kbcwRyxVu_srprAAoTYzujnBvA6jU-0SMxkRgOhGjYJSUNGKvw24EQwJa1jG5RM76mQD2P22AKSq8qSD94LddoXGdKDO74eQVYl0RhuqOMFqdrEZpp1p4bIR6_N8VeSJDHr7UCuo8FiabkSHrkq7tsNvRP-yI-bnpQv4EayPBh7YwHVX3hYdRbhxaBvgJENgCuiEPb35Q2-Z6w6ujjiKUAK2VSbCFpENlEfcHNsjDeY7RCvFlwlCoHj1IeiNIaFTE9yXFu3aEWlExe3LzHv6PZyunEi88QJSXKSh56Um0e0eEg05rMv-VBM83cAqc5POmZnTP1vUdZO8fQv3NFLZ-xU6e1WQVxVPi5Cyqk4jYhGf1Y9t98N654y0tVvw74qNIsTrB-8bGS0Uhfe24oBEWmzObvBY3zhtT1d42myGUJv4pMTU6yPoS83zhPKm3LbUDEpBA1hvvc_09jHk7vUEAuFB-UAZzlht2C1yklzQ",
-            "update_time": 1688351991,
-            "token_expires_time": 1688355491,
-            "create_time": 1683702488
-        }
-    return {}
-
-
-class BaseTask:
-    AmzAdClientClass = BaseClient
-
-    def __init__(self, profile_id: str):
-        self.shop_info = query_shop_info(profile_id)
-        self.ad_cil = self.AmzAdClientClass(
-            profile_id=profile_id,
-            refresh_token=self.shop_info["refresh_token"],
-            data_path=DATA_PATH, **AWS_LWA_CLIENT)
-
-    def do(self, task_info: dict):
-        record = task_info["record"]
-        iter_records = getattr(self.ad_cil, f"iter_{record}", None)
-        change_func = getattr(self, f"change_{record}", None)
-        ad_api_params = task_info.get("params")
-        if ad_api_params is None:
-            records_iterator = iter_records()
-        else:
-            records_iterator = iter_records(ad_api_params)
-        for data in records_iterator:
-            if change_func:
-                data = change_func(data)
-            self.to_mysql(record, data)
-            logger.info(data)
-
-    def batch_do(self, task_info: dict):
-        """
-        适用于使用pandas等进行批处理
-        @param task_info:
-        @return:
-        """
-        pass
-
-    def to_mysql(self, record: str, data: [dict, List[dict]]):
-        asj_api.create(f"{ASJ_URL_BASE}{record}/", data)

sync_amz_data/tasks/account.py

@@ -1,19 +0,0 @@
-from ._base import BaseTask
-from sync_amz_data.public import AccountClient
-from sync_amz_data.tools import timestamp2utc_dt
-
-
-class AccountTask(BaseTask):
-    AmzAdClientClass = AccountClient
-
-    def change_portfolios(self, data: dict):
-        data["shop"] = self.shop_info["profile_id"]
-        for key in ["lastUpdatedDate", "creationDate"]:
-            if key in data:
-                data[key] = timestamp2utc_dt(data[key])
-
-        budget: dict = data.pop("budget", None)
-        if budget:
-            for key, val in budget.items():
-                data[f"budget_{key}"] = val
-        return data

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbads.py

@@ -1,106 +0,0 @@
-import requests
-from urllib.parse import urljoin
-from sync_amz_data.public.amz_ad_client import SBClient
-from sync_amz_data.settings import AWS_LWA_CLIENT
-import pandas as pd
-import json
-
-
-class RateLimitError(Exception):
-    def __init__(self, retry_after: str = None):
-        self.retry_after = retry_after
-
-
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
-    resp = requests.session().request(
-        method=method,
-        url=urljoin(ADS, url_path),
-        headers=head,
-        params=params,
-        json=body,
-    )
-    if resp.status_code == 429:
-        raise RateLimitError(resp.headers.get("Retry-After"))
-    if resp.status_code >= 400:
-        raise Exception(resp.text)
-    return resp.json()
-
-
-class SbAds:
-    def __init__(self, profile_id, campaignId: list = None):
-        self.profile_id = profile_id
-        self.portfolioId = campaignId
-        self.re_url_path = "api/ad_manage/profiles/"
-        self.upcreate_url_path = "api/ad_manage/sbads/updata/"
-        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
-        self.refresh_token = self.get_refresh_token()
-        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
-        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
-        self.AWS_CREDENTIALS = {
-            'lwa_client_id': self.lwa_client_id,
-            'lwa_client_secret': self.lwa_client_secret,
-            'refresh_token': self.refresh_token,
-            'profile_id': self.profile_id
-        }
-
-    def get_refresh_token(self):
-        params = {'profile_id': self.profile_id}
-        heads = self.heads
-        url_path = self.re_url_path
-        tem = request(url_path=url_path, head=heads, params=params)
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            out = _[0].get('refresh_token')
-        else:
-            out = None
-        return out
-
-    def get_sbads_data(self):
-        tem = SBClient(**self.AWS_CREDENTIALS)
-        list_ads = tem.iter_ads(**{"includeExtendedDataFields": True})
-        df_ads = pd.json_normalize(list(list_ads))
-        return df_ads
-
-    def dataconvert(self):
-        df = self.get_sbads_data()
-        df['extendedData.creationDate'] = pd.to_datetime(df['extendedData.creationDate'],
-                                                             unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
-        df['extendedData.lastUpdateDate'] = pd.to_datetime(df['extendedData.lastUpdateDate'],
-                                                               unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
-        col = ['adId', 'campaignId', 'landingPage.asins', 'landingPage.pageType', 'landingPage.url', 'name', 'state',
-               'adGroupId', 'creative.asins', 'creative.type', 'extendedData.creationDate',
-               'extendedData.lastUpdateDate', 'extendedData.servingStatus', 'extendedData.servingStatusDetails']
-        for i in col:
-            if i not in df.columns:
-                df['{}'.format(i)] = None
-        df_new = df[col].copy()
-        old = pd.DataFrame(data=[], columns=col)
-        tem = pd.concat([old, df_new]).reset_index()
-        tem.drop(columns='index', inplace=True)
-        tem.columns = [i.replace(".", "_") for i in tem.columns]
-        tem.rename(columns={
-            'adGroupId': 'adGroup',
-            'campaignId': 'campaign',
-            'extendedData_creationDate': 'creationDateTime',
-            'extendedData_lastUpdateDate': 'lastUpdateDateTime',
-            'extendedData_servingStatus': 'servingStatus',
-            'extendedData_servingStatusDetails': 'servingStatusDetails',
-            'name': 'adName',
-        }, inplace=True)
-        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
-        return json_data
-
-    def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
-
-
-if __name__ == '__main__':
-    a = SbAds(profile_id="3006125408623189")
-    out = a.updata_create()
-    # out = a.dataconvert()
-    print(out)

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbcampaign.py

@@ -1,123 +0,0 @@
-import requests
-from urllib.parse import urljoin
-from sync_amz_data.public.amz_ad_client import SBClient
-from sync_amz_data.settings import AWS_LWA_CLIENT
-import pandas as pd
-import json
-
-
-class RateLimitError(Exception):
-    def __init__(self, retry_after: str = None):
-        self.retry_after = retry_after
-
-
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
-    resp = requests.session().request(
-        method=method,
-        url=urljoin(ADS, url_path),
-        headers=head,
-        params=params,
-        json=body,
-    )
-    if resp.status_code == 429:
-        raise RateLimitError(resp.headers.get("Retry-After"))
-    if resp.status_code >= 400:
-        raise Exception(resp.text)
-    return resp.json()
-
-
-class SbCampaign:
-    def __init__(self, profile_id, portfolioId: list = None):
-        self.profile_id = profile_id
-        self.portfolioId = portfolioId
-        self.re_url_path = "api/ad_manage/profiles/"
-        self.upcreate_url_path = "api/ad_manage/sbcampaigns/updata/"
-        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
-        self.refresh_token = self.get_refresh_token()
-        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
-        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
-        self.AWS_CREDENTIALS = {
-            'lwa_client_id': self.lwa_client_id,
-            'lwa_client_secret': self.lwa_client_secret,
-            'refresh_token': self.refresh_token,
-            'profile_id': self.profile_id
-        }
-
-    def get_refresh_token(self):
-        params = {'profile_id': self.profile_id}
-        heads = self.heads
-        url_path = self.re_url_path
-        tem = request(url_path=url_path, head=heads, params=params)
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            out = _[0].get('refresh_token')
-        else:
-            out = None
-        return out
-
-    def get_sbcampaign_data(self):
-        tem = SBClient(**self.AWS_CREDENTIALS)
-        if self.portfolioId is None:
-            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True,
-                                                   "stateFilter": {
-                                                       "include": ["ARCHIVED", "PAUSED", "ENABLED"]
-                                                   }})
-        else:
-            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True,
-                                                   "portfolioIdFilter": {"include": self.portfolioId},
-                                                   "stateFilter": {
-                                                       "include": ["ARCHIVED", "PAUSED", "ENABLED"]
-                                                   }})
-        df_campaigns = pd.json_normalize(list(list_campaigns))
-        return df_campaigns
-
-    def dataconvert(self):
-        df = self.get_sbcampaign_data()
-        df['profile_id'] = self.profile_id
-        df['extendedData.creationDate'] = pd.to_datetime(df['extendedData.creationDate'], unit='ms').dt.strftime(
-            '%Y-%m-%d %H:%M:%S')
-        df['extendedData.lastUpdateDate'] = pd.to_datetime(df['extendedData.lastUpdateDate'], unit='ms').dt.strftime(
-            '%Y-%m-%d %H:%M:%S')
-        col = ['budgetType', 'ruleBasedBudget.isProcessing', 'ruleBasedBudget.applicableRuleName',
-               'ruleBasedBudget.value', 'ruleBasedBudget.applicableRuleId', 'brandEntityId',
-               'isMultiAdGroupsEnabled', 'goal', 'bidding.bidOptimization', 'bidding.bidAdjustmentsByShopperSegment',
-               'bidding.bidAdjustmentsByPlacement', 'bidding.bidOptimizationStrategy', 'endDate',
-               'campaignId', 'productLocation', 'portfolioId', 'costType', 'smartDefault', 'name', 'state',
-               'startDate', 'budget', 'extendedData.servingStatus', 'extendedData.lastUpdateDate',
-               'extendedData.creationDate', 'extendedData.servingStatusDetails']
-        old = pd.DataFrame(data=[], columns=col)
-        tem = pd.concat([old, df]).reset_index()
-        tem.drop(columns='index', inplace=True)
-        tem.columns = [i.replace(".", "_") for i in tem.columns]
-        tem.rename(columns={
-            'name': 'campaignName',
-            'portfolioId': 'portfolio',
-            'ruleBasedBudget_isProcessing': 'rbb_isProcessing',
-            'ruleBasedBudget_applicableRuleName': 'rbb_appRuleName',
-            'ruleBasedBudget_value': 'rbb_value',
-            'ruleBasedBudget_applicableRuleId': 'rbb_appRuleId',
-            'bidding_bidOptimization': 'bidOptimization',
-            'bidding_bidAdjustmentsByShopperSegment': 'bidShopperSegment',
-            'bidding_bidAdjustmentsByPlacement': 'bidPlacement',
-            'bidding_bidOptimizationStrategy': 'bidOptimizationStrategy',
-            'extendedData_lastUpdateDate': 'lastUpdateDate',
-            'extendedData_servingStatus': 'servingStatus',
-            'extendedData_servingStatusDetails': 'servingStatusDetails',
-            'extendedData_creationDate': 'creationDate'}, inplace=True)
-        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
-        return json_data
-
-    def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
-
-
-if __name__ == '__main__':
-    a = SbCampaign(profile_id="3006125408623189")
-    out = a.updata_create()
-    # out = a.dataconvert()
-    print(out)

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbgroup.py

@@ -1,104 +0,0 @@
-import requests
-from urllib.parse import urljoin
-from sync_amz_data.public.amz_ad_client import SBClient
-from sync_amz_data.settings import AWS_LWA_CLIENT
-import pandas as pd
-import json
-
-
-class RateLimitError(Exception):
-    def __init__(self, retry_after: str = None):
-        self.retry_after = retry_after
-
-
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
-    resp = requests.session().request(
-        method=method,
-        url=urljoin(ADS, url_path),
-        headers=head,
-        params=params,
-        json=body,
-    )
-    if resp.status_code == 429:
-        raise RateLimitError(resp.headers.get("Retry-After"))
-    if resp.status_code >= 400:
-        raise Exception(resp.text)
-    return resp.json()
-
-
-class SbGroup:
-    def __init__(self, profile_id, campaignId: list = None):
-        self.profile_id = profile_id
-        self.portfolioId = campaignId
-        self.re_url_path = "api/ad_manage/profiles/"
-        self.upcreate_url_path = "api/ad_manage/sbgroups/updata/"
-        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
-        self.refresh_token = self.get_refresh_token()
-        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
-        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
-        self.AWS_CREDENTIALS = {
-            'lwa_client_id': self.lwa_client_id,
-            'lwa_client_secret': self.lwa_client_secret,
-            'refresh_token': self.refresh_token,
-            'profile_id': self.profile_id
-        }
-
-    def get_refresh_token(self):
-        params = {'profile_id': self.profile_id}
-        heads = self.heads
-        url_path = self.re_url_path
-        tem = request(url_path=url_path, head=heads, params=params)
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            out = _[0].get('refresh_token')
-        else:
-            out = None
-        return out
-
-    def get_spgroup_data(self):
-        tem = SBClient(**self.AWS_CREDENTIALS)
-        list_group = tem.iter_adGroups(**{"includeExtendedDataFields": True,
-                                          "stateFilter": {
-                                              "include": ["ARCHIVED", "PAUSED", "ENABLED"]
-                                          }})
-        df_group = pd.json_normalize(list(list_group))
-        return df_group
-    #----------------------------
-
-    def dataconvert(self):
-        df = self.get_spgroup_data()
-        df['extendedData.creationDate'] = pd.to_datetime(
-            df['extendedData.creationDate'], unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
-        df['extendedData.lastUpdateDate'] = pd.to_datetime(
-            df['extendedData.lastUpdateDate'], unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
-        col = ['adGroupId', 'campaignId', 'name', 'state', 'extendedData.creationDate',
-               'extendedData.lastUpdateDate', 'extendedData.servingStatus', 'extendedData.servingStatusDetails']
-        old = pd.DataFrame(data=[], columns=col)
-        tem = pd.concat([old, df]).reset_index()
-        tem.drop(columns='index', inplace=True)
-        tem.columns = [i.replace(".", "_") for i in tem.columns]
-        tem.rename(columns={
-            'name': 'adGroupName',
-            'campaignId': 'campaign',
-            'extendedData_creationDate': 'creationDateTime',
-            'extendedData_lastUpdateDate': 'lastUpdateDateTime',
-            'extendedData_servingStatus': 'servingStatus',
-            'extendedData_servingStatusDetails': 'servingStatusDetails'
-        }, inplace=True)
-        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
-        return json_data
-
-    def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
-
-
-if __name__ == '__main__':
-    a = SbGroup(profile_id="3006125408623189")
-    out = a.updata_create()
-    # out = a.dataconvert()
-    print(out)

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbkeyword_v3.py

@@ -1,97 +0,0 @@
-import requests
-from urllib.parse import urljoin
-from sync_amz_data.public.amz_ad_client import SBClient
-from sync_amz_data.settings import AWS_LWA_CLIENT
-import pandas as pd
-import json
-
-
-class RateLimitError(Exception):
-    def __init__(self, retry_after: str = None):
-        self.retry_after = retry_after
-
-
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
-    resp = requests.session().request(
-        method=method,
-        url=urljoin(ADS, url_path),
-        headers=head,
-        params=params,
-        json=body,
-    )
-    if resp.status_code == 429:
-        raise RateLimitError(resp.headers.get("Retry-After"))
-    if resp.status_code >= 400:
-        raise Exception(resp.text)
-    return resp.json()
-
-
-class SpKeyword:
-    def __init__(self, profile_id, campaignId: list = None):
-        self.profile_id = profile_id
-        self.portfolioId = campaignId
-        self.re_url_path = "api/ad_manage/profiles/"
-        self.upcreate_url_path = "api/ad_manage/sbkeywords/updata/"
-        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
-        self.refresh_token = self.get_refresh_token()
-        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
-        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
-        self.AWS_CREDENTIALS = {
-            'lwa_client_id': self.lwa_client_id,
-            'lwa_client_secret': self.lwa_client_secret,
-            'refresh_token': self.refresh_token,
-            'profile_id': self.profile_id
-        }
-
-    def get_refresh_token(self):
-        params = {'profile_id': self.profile_id}
-        heads = self.heads
-        url_path = self.re_url_path
-        tem = request(url_path=url_path, head=heads, params=params)
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            out = _[0].get('refresh_token')
-        else:
-            out = None
-        return out
-
-    def get_sbkeywords_data(self):
-        tem = SBClient(**self.AWS_CREDENTIALS)
-        _ = tem.iter_keywords()
-        list_sbkeywords = [row for i in _ for row in i]
-        df_sbkeywords = pd.json_normalize(list(list_sbkeywords))
-        return df_sbkeywords
-
-    def dataconvert(self):
-        df = self.get_sbkeywords_data()
-        col = ['keywordId', 'adGroupId', 'campaignId', 'keywordText',
-               'nativeLanguageKeyword', 'matchType', 'state', 'bid']
-        old = pd.DataFrame(data=[], columns=col)
-        for i in col:
-            if i not in df.columns:
-                df['{}'.format(i)] = None
-        df_new = df[col].copy()
-        tem = pd.concat([old, df_new]).reset_index()
-        tem.drop(columns='index', inplace=True)
-        tem.columns = [i.replace(".", "_") for i in tem.columns]
-        tem.rename(columns={
-            'adGroupId': 'adGroup',
-            'campaignId': 'campaign',
-        }, inplace=True)
-        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
-        return json_data
-
-    def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
-
-
-if __name__ == '__main__':
-    a = SpKeyword(profile_id="3006125408623189")
-    out = a.updata_create()
-    # out = a.dataconvert()
-    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sp_budget_recommendation.py

@@ -1,128 +0,0 @@
-import requests
-from urllib.parse import urljoin
-from sync_amz_data.public.amz_ad_client import SPClient
-from sync_amz_data.settings import AWS_LWA_CLIENT
-import pandas as pd
-import json
-
-
-class RateLimitError(Exception):
-    def __init__(self, retry_after: str = None):
-        self.retry_after = retry_after
-
-
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
-    resp = requests.session().request(
-        method=method,
-        url=urljoin(ADS, url_path),
-        headers=head,
-        params=params,
-        json=body,
-    )
-    if resp.status_code == 429:
-        raise RateLimitError(resp.headers.get("Retry-After"))
-    if resp.status_code >= 400:
-        raise Exception(resp.text)
-    return resp.json()
-
-
-class SpBudgetRecommendation:
-    def __init__(self, profile_id, portfolioId: list = None):
-        self.profile_id = profile_id
-        self.portfolioId = portfolioId
-        self.re_url_path = "api/ad_manage/profiles/"
-        self.camid_url_path = "api/ad_manage/spcampaigns/?updata=1&limit=9999"
-        self.upcreate_url_path = "api/ad_manage/spbudgetrecommendation/updata/"
-        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
-        self.refresh_token = self.get_refresh_token()
-        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
-        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
-        self.AWS_CREDENTIALS = {
-            'lwa_client_id': self.lwa_client_id,
-            'lwa_client_secret': self.lwa_client_secret,
-            'refresh_token': self.refresh_token,
-            'profile_id': self.profile_id
-        }
-
-    def get_refresh_token(self):
-        params = {'profile_id': self.profile_id}
-        heads = self.heads
-        url_path = self.re_url_path
-        tem = request(url_path=url_path, head=heads, params=params)
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            out = _[0].get('refresh_token')
-        else:
-            out = None
-        return out
-
-    def get_campaignid(self):
-        heads = self.heads
-        url_path = self.camid_url_path
-        tem = request(url_path=url_path, head=heads)
-        out = []
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            for i in _:
-                out.append(i['campaignId'])
-        return out
-
-    def get_budgetrecommendation_data(self):
-        tem = SPClient(**self.AWS_CREDENTIALS)
-        camid_list = self.get_campaignid()
-        list_tem = tem.iter_budgetrecommendation(camid_list)
-        df_spbudgetrecommendation = pd.json_normalize(list(list_tem))
-        return df_spbudgetrecommendation
-
-    def dataconvert(self):
-        df = self.get_budgetrecommendation_data()
-        df.drop(columns='index', inplace=True)
-        df['sevenDaysMissedOpportunities.startDate'] = pd.to_datetime(
-            df['sevenDaysMissedOpportunities.startDate'], format='%Y%m%d').dt.strftime(
-            '%Y-%m-%d')
-        df['sevenDaysMissedOpportunities.endDate'] = pd.to_datetime(
-            df['sevenDaysMissedOpportunities.endDate'], format='%Y%m%d').dt.strftime(
-            '%Y-%m-%d')
-        col = ['campaignId', 'suggestedBudget', 'budgetRuleRecommendation',
-               'sevenDaysMissedOpportunities.startDate',
-               'sevenDaysMissedOpportunities.endDate',
-               'sevenDaysMissedOpportunities.percentTimeInBudget',
-               'sevenDaysMissedOpportunities.estimatedMissedImpressionsLower',
-               'sevenDaysMissedOpportunities.estimatedMissedImpressionsUpper',
-               'sevenDaysMissedOpportunities.estimatedMissedClicksLower',
-               'sevenDaysMissedOpportunities.estimatedMissedClicksUpper',
-               'sevenDaysMissedOpportunities.estimatedMissedSalesLower',
-               'sevenDaysMissedOpportunities.estimatedMissedSalesUpper']
-        old = pd.DataFrame(data=[], columns=col)
-        tem = pd.concat([old, df]).reset_index()
-        tem.drop(columns='index', inplace=True)
-        tem.columns = [i.replace(".", "_") for i in tem.columns]
-        tem.rename(columns={
-            'campaignId': 'campaign',
-            'sevenDaysMissedOpportunities_startDate': 'startDate',
-            'sevenDaysMissedOpportunities_endDate': 'endDate',
-            'sevenDaysMissedOpportunities_percentTimeInBudget': 'percentTimeInBudget',
-            'sevenDaysMissedOpportunities_estimatedMissedImpressionsLower': 'MissedImpressionsLower',
-            'sevenDaysMissedOpportunities_estimatedMissedImpressionsUpper': 'MissedImpressionsUpper',
-            'sevenDaysMissedOpportunities_estimatedMissedClicksLower': 'MissedClicksLower',
-            'sevenDaysMissedOpportunities_estimatedMissedClicksUpper': 'MissedClicksUpper',
-            'sevenDaysMissedOpportunities_estimatedMissedSalesLower': 'MissedSalesLower',
-            'sevenDaysMissedOpportunities_estimatedMissedSalesUpper': 'MissedSalesUpper',
-        }, inplace=True)
-        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
-        return json_data
-
-    def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
-
-
-if __name__ == '__main__':
-    a = SpBudgetRecommendation(profile_id="3006125408623189")
-    out = a.updata_create()
-    # out = a.dataconvert()
-    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sp_targetsbid_recommendations.py

@@ -1,132 +0,0 @@
-import requests
-from urllib.parse import urljoin
-from sync_amz_data.public.amz_ad_client import SPClient
-from sync_amz_data.settings import AWS_LWA_CLIENT
-import pandas as pd
-import json
-
-pd.set_option('display.max_columns', None)
-# 显示所有行
-pd.set_option('display.max_rows', None)
-
-
-class RateLimitError(Exception):
-    def __init__(self, retry_after: str = None):
-        self.retry_after = retry_after
-
-
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
-    resp = requests.session().request(
-        method=method,
-        url=urljoin(ADS, url_path),
-        headers=head,
-        params=params,
-        json=body,
-    )
-    if resp.status_code == 429:
-        raise RateLimitError(resp.headers.get("Retry-After"))
-    if resp.status_code >= 400:
-        raise Exception(resp.text)
-    return resp.json()
-
-
-class SpTargetsBidRecommendations:
-    def __init__(self, profile_id):
-        self.profile_id = profile_id
-        self.re_url_path = "api/ad_manage/profiles/"
-        self.cgk_url_path = "api/ad_manage/sptbrkeywords/"
-        self.upcreate_url_path = "api/ad_manage/sptargetsbidrecommendation/updata/"
-        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
-        self.refresh_token = self.get_refresh_token()
-        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
-        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
-        self.AWS_CREDENTIALS = {
-            'lwa_client_id': self.lwa_client_id,
-            'lwa_client_secret': self.lwa_client_secret,
-            'refresh_token': self.refresh_token,
-            'profile_id': self.profile_id
-        }
-
-    def get_refresh_token(self):
-        params = {'profile_id': self.profile_id}
-        heads = self.heads
-        url_path = self.re_url_path
-        tem = request(url_path=url_path, head=heads, params=params)
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            out = _[0].get('refresh_token')
-        else:
-            out = None
-        return out
-
-    def get_arg(self):
-        heads = self.heads
-        url_path = self.cgk_url_path
-        data = []
-        page = 1
-        params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
-        tem = request(url_path=url_path, head=heads, params=params)
-        data.extend(tem.get('data'))
-        while tem.get('is_next') is True:
-            page += 1
-            params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
-            tem = request(url_path=url_path, head=heads, params=params)
-            data.extend(tem.get('data'))
-        _ = pd.json_normalize(data)
-        df = _.copy()
-        df.rename(columns={'keywordText': 'value', 'matchType': 'type'}, inplace=True)
-        df['targetingExpressions'] = df[['value', 'type', 'keywordId']].apply(lambda x: x.to_dict(), axis=1)
-        df_grouped = df.groupby(['campaignId', 'adGroupId']).agg({'targetingExpressions': list}).reset_index()
-        return df_grouped
-
-    def get_sptargetsbidrecommendation_data(self):
-        tem = SPClient(**self.AWS_CREDENTIALS)
-        df_arg = self.get_arg()
-        data_json = df_arg.to_json(orient='records')
-        list_arg = json.loads(data_json)
-        out_df = pd.DataFrame()
-        for i in list_arg:
-            k_id_text_df = pd.DataFrame.from_dict(i['targetingExpressions'])
-            list_sptbr = tem.iter_adgroup_bidrecommendation(**i)
-            list_outdata = list(list_sptbr)
-            if len(list_outdata) > 0:
-                out_data = []
-                for j in list_outdata:
-                    if j.get('theme') == "CONVERSION_OPPORTUNITIES":
-                        data = j.get('bidRecommendationsForTargetingExpressions')
-                        out_data.extend(data)
-                temtest = pd.json_normalize(out_data)
-                temtest.rename(columns={'targetingExpression.value': 'value'}, inplace=True)
-                temtest.rename(columns={'targetingExpression.type': 'type'}, inplace=True)
-                df_tem = pd.merge(left=temtest, right=k_id_text_df, on=['value', 'type'], how='left')
-                out_df = pd.concat([out_df, df_tem])
-        out_df['suggestedBid'] = out_df.apply(
-            lambda row: row['bidValues'][1]['suggestedBid'] if len(row['bidValues']) > 0 else None,
-            axis=1)
-        out_df['suggestedBid_lower'] = out_df.apply(
-            lambda row: row['bidValues'][0]['suggestedBid'] if len(row['bidValues']) > 0 else None,
-            axis=1)
-        out_df['suggestedBid_upper'] = out_df.apply(
-            lambda row: row['bidValues'][2]['suggestedBid'] if len(row['bidValues']) > 0 else None,
-            axis=1)
-        out_df.drop(labels='bidValues', inplace=True, axis=1)
-        out_df.drop(labels='value', inplace=True, axis=1)
-        out_df.rename(columns={'type': 'targetingExpression_type',
-                               'keywordId': 'keyword'}, inplace=True)
-        json_data = json.loads(out_df.to_json(orient='records', force_ascii=False))
-        return json_data
-
-    def updata_create(self):
-        body = self.get_sptargetsbidrecommendation_data()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
-
-
-if __name__ == '__main__':
-    a = SpTargetsBidRecommendations(profile_id="3006125408623189")
-    # out = a.get_sptargetsbidrecommendation_data()
-    out = a.updata_create()
-    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spads.py

@@ -1,100 +0,0 @@
-import requests
-from urllib.parse import urljoin
-from sync_amz_data.public.amz_ad_client import SPClient
-from sync_amz_data.settings import AWS_LWA_CLIENT
-import pandas as pd
-import json
-
-
-class RateLimitError(Exception):
-    def __init__(self, retry_after: str = None):
-        self.retry_after = retry_after
-
-
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
-    resp = requests.session().request(
-        method=method,
-        url=urljoin(ADS, url_path),
-        headers=head,
-        params=params,
-        json=body,
-    )
-    if resp.status_code == 429:
-        raise RateLimitError(resp.headers.get("Retry-After"))
-    if resp.status_code >= 400:
-        raise Exception(resp.text)
-    return resp.json()
-
-
-class SpAds:
-    def __init__(self, profile_id, campaignId: list = None):
-        self.profile_id = profile_id
-        self.portfolioId = campaignId
-        self.re_url_path = "api/ad_manage/profiles/"
-        self.upcreate_url_path = "api/ad_manage/spads/updata/"
-        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
-        self.refresh_token = self.get_refresh_token()
-        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
-        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
-        self.AWS_CREDENTIALS = {
-            'lwa_client_id': self.lwa_client_id,
-            'lwa_client_secret': self.lwa_client_secret,
-            'refresh_token': self.refresh_token,
-            'profile_id': self.profile_id
-        }
-
-    def get_refresh_token(self):
-        params = {'profile_id': self.profile_id}
-        heads = self.heads
-        url_path = self.re_url_path
-        tem = request(url_path=url_path, head=heads, params=params)
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            out = _[0].get('refresh_token')
-        else:
-            out = None
-        return out
-
-    def get_spads_data(self):
-        tem = SPClient(**self.AWS_CREDENTIALS)
-        list_ads = tem.iter_ads(**{"includeExtendedDataFields": True})
-        df_ads = pd.json_normalize(list(list_ads))
-        return df_ads
-
-    def dataconvert(self):
-        df = self.get_spads_data()
-        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
-            '%Y-%m-%d %H:%M:%S')
-        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
-            '%Y-%m-%d %H:%M:%S')
-        col = ['adId', 'campaignId', 'customText', 'asin', 'state', 'sku', 'adGroupId', 'extendedData.creationDateTime',
-               'extendedData.lastUpdateDateTime', 'extendedData.servingStatus', 'extendedData.servingStatusDetails']
-        old = pd.DataFrame(data=[], columns=col)
-        tem = pd.concat([old, df]).reset_index()
-        tem.drop(columns='index', inplace=True)
-        tem.columns = [i.replace(".", "_") for i in tem.columns]
-        tem.rename(columns={
-            'adGroupId': 'adGroup',
-            'campaignId': 'campaign',
-            'extendedData_creationDateTime': 'creationDateTime',
-            'extendedData_lastUpdateDateTime': 'lastUpdateDateTime',
-            'extendedData_servingStatus': 'servingStatus',
-            'extendedData_servingStatusDetails': 'servingStatusDetails'
-        }, inplace=True)
-        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
-        return json_data
-
-    def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
-
-
-if __name__ == '__main__':
-    a = SpAds(profile_id="3006125408623189")
-    out = a.updata_create()
-    # out = a.dataconvert()
-    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spcampaign.py

@@ -1,114 +0,0 @@
-import requests
-from urllib.parse import urljoin
-from sync_amz_data.public.amz_ad_client import SPClient
-from sync_amz_data.settings import AWS_LWA_CLIENT
-import pandas as pd
-import json
-
-
-class RateLimitError(Exception):
-    def __init__(self, retry_after: str = None):
-        self.retry_after = retry_after
-
-
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
-    resp = requests.session().request(
-        method=method,
-        url=urljoin(ADS, url_path),
-        headers=head,
-        params=params,
-        json=body,
-    )
-    if resp.status_code == 429:
-        raise RateLimitError(resp.headers.get("Retry-After"))
-    if resp.status_code >= 400:
-        raise Exception(resp.text)
-    return resp.json()
-
-
-class SpCampaign:
-    def __init__(self, profile_id, portfolioId: list = None):
-        self.profile_id = profile_id
-        self.portfolioId = portfolioId
-        self.re_url_path = "api/ad_manage/profiles/"
-        self.upcreate_url_path = "api/ad_manage/spcampaigns/updata/?updata=1"
-        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
-        self.refresh_token = self.get_refresh_token()
-        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
-        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
-        self.AWS_CREDENTIALS = {
-            'lwa_client_id': self.lwa_client_id,
-            'lwa_client_secret': self.lwa_client_secret,
-            'refresh_token': self.refresh_token,
-            'profile_id': self.profile_id
-        }
-
-    def get_refresh_token(self):
-        params = {'profile_id': self.profile_id}
-        heads = self.heads
-        url_path = self.re_url_path
-        tem = request(url_path=url_path, head=heads, params=params)
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            out = _[0].get('refresh_token')
-        else:
-            out = None
-        return out
-
-    def get_spcampaign_data(self):
-        tem = SPClient(**self.AWS_CREDENTIALS)
-        if self.portfolioId is None:
-            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True})
-        else:
-            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True,
-                                                   "portfolioIdFilter": {"include": self.portfolioId}
-                                                   })
-        df_campaigns = pd.json_normalize(list(list_campaigns))
-        return df_campaigns
-
-    def dataconvert(self):
-        df = self.get_spcampaign_data()
-        df['profile_id'] = self.profile_id
-        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
-            '%Y-%m-%d %H:%M:%S')
-        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
-            '%Y-%m-%d %H:%M:%S')
-        col = ['campaignId', 'endDate', 'name', 'startDate', 'state', 'targetingType',
-               'budget.budget', 'budget.budgetType', 'budget.effectiveBudget',
-               'dynamicBidding.placementBidding', 'extendedData.lastUpdateDateTime',
-               'dynamicBidding.strategy', 'portfolioId', 'profile_id',
-               'extendedData.servingStatus', 'extendedData.servingStatusDetails',
-               'extendedData.creationDateTime']
-        old = pd.DataFrame(data=[], columns=col)
-        tem = pd.concat([old, df]).reset_index()
-        tem.drop(columns='index', inplace=True)
-        tem.columns = [i.replace(".", "_") for i in tem.columns]
-        tem.rename(columns={
-            'name': 'campaignName',
-            'portfolioId': 'portfolio',
-            'dynamicBidding_placementBidding': 'dynBidPlacement',
-            'dynamicBidding_strategy': 'dynBidStrategy',
-            'budget_budgetType': 'budgetType',
-            'budget_budget': 'budget',
-            'budget_effectiveBudget': 'effectiveBudget',
-            'extendedData_lastUpdateDateTime': 'lastUpdateDateTime',
-            'extendedData_servingStatus': 'servingStatus',
-            'extendedData_servingStatusDetails': 'servingStatusDetails',
-            'extendedData_creationDateTime': 'creationDateTime'}, inplace=True)
-        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
-        return json_data
-
-    def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
-
-
-if __name__ == '__main__':
-    a = SpCampaign(profile_id="3006125408623189")
-    out = a.updata_create()
-    # out = a.dataconvert()
-    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spgroup.py

@@ -1,103 +0,0 @@
-import requests
-from urllib.parse import urljoin
-from sync_amz_data.public.amz_ad_client import SPClient
-from sync_amz_data.settings import AWS_LWA_CLIENT
-import pandas as pd
-import json
-
-
-class RateLimitError(Exception):
-    def __init__(self, retry_after: str = None):
-        self.retry_after = retry_after
-
-
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
-    resp = requests.session().request(
-        method=method,
-        url=urljoin(ADS, url_path),
-        headers=head,
-        params=params,
-        json=body,
-    )
-    if resp.status_code == 429:
-        raise RateLimitError(resp.headers.get("Retry-After"))
-    if resp.status_code >= 400:
-        raise Exception(resp.text)
-    return resp.json()
-
-
-class SpGroup:
-    def __init__(self, profile_id, campaignId: list = None):
-        self.profile_id = profile_id
-        self.portfolioId = campaignId
-        self.re_url_path = "api/ad_manage/profiles/"
-        self.upcreate_url_path = "api/ad_manage/spgroups/updata/?updata=1"
-        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
-        self.refresh_token = self.get_refresh_token()
-        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
-        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
-        self.AWS_CREDENTIALS = {
-            'lwa_client_id': self.lwa_client_id,
-            'lwa_client_secret': self.lwa_client_secret,
-            'refresh_token': self.refresh_token,
-            'profile_id': self.profile_id
-        }
-
-    def get_refresh_token(self):
-        params = {'profile_id': self.profile_id}
-        heads = self.heads
-        url_path = self.re_url_path
-        tem = request(url_path=url_path, head=heads, params=params)
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            out = _[0].get('refresh_token')
-        else:
-            out = None
-        return out
-
-    def get_spgroup_data(self):
-        tem = SPClient(**self.AWS_CREDENTIALS)
-        list_group = tem.iter_adGroups(**{"includeExtendedDataFields": True})
-        df_group = pd.json_normalize(list(list_group))
-        return df_group
-    #----------------------------
-
-    def dataconvert(self):
-        df = self.get_spgroup_data()
-        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
-            '%Y-%m-%d %H:%M:%S')
-        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
-            '%Y-%m-%d %H:%M:%S')
-        col = ['adGroupId', 'campaignId', 'defaultBid', 'name', 'state', 'extendedData.creationDateTime',
-               'extendedData.lastUpdateDateTime', 'extendedData.servingStatus', 'extendedData.servingStatusDetails']
-        old = pd.DataFrame(data=[], columns=col)
-        tem = pd.concat([old, df]).reset_index()
-        tem.drop(columns='index', inplace=True)
-        tem.columns = [i.replace(".", "_") for i in tem.columns]
-        tem.rename(columns={
-            'name': 'adGroupName',
-            'campaignId': 'campaign',
-            'extendedData_creationDateTime': 'creationDateTime',
-            'extendedData_lastUpdateDateTime': 'lastUpdateDateTime',
-            'extendedData_servingStatus': 'servingStatus',
-            'extendedData_servingStatusDetails': 'servingStatusDetails'
-        }, inplace=True)
-        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
-        return json_data
-
-    def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
-
-
-if __name__ == '__main__':
-    a = SpGroup(profile_id="3006125408623189")
-    out = a.updata_create()
-    # out = a.dataconvert()
-    print(out)
-
-

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spkeyword.py

@@ -1,102 +0,0 @@
-import requests
-from urllib.parse import urljoin
-from sync_amz_data.public.amz_ad_client import SPClient
-from sync_amz_data.settings import AWS_LWA_CLIENT
-import pandas as pd
-import json
-
-
-class RateLimitError(Exception):
-    def __init__(self, retry_after: str = None):
-        self.retry_after = retry_after
-
-
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
-    resp = requests.session().request(
-        method=method,
-        url=urljoin(ADS, url_path),
-        headers=head,
-        params=params,
-        json=body,
-    )
-    if resp.status_code == 429:
-        raise RateLimitError(resp.headers.get("Retry-After"))
-    if resp.status_code >= 400:
-        raise Exception(resp.text)
-    return resp.json()
-
-
-class SpKeyword:
-    def __init__(self, profile_id, campaignId: list = None):
-        self.profile_id = profile_id
-        self.campaignId = campaignId
-        self.re_url_path = "api/ad_manage/profiles/"
-        self.upcreate_url_path = "api/ad_manage/spkeywords/updata/?updata=1"
-        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
-        self.refresh_token = self.get_refresh_token()
-        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
-        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
-        self.AWS_CREDENTIALS = {
-            'lwa_client_id': self.lwa_client_id,
-            'lwa_client_secret': self.lwa_client_secret,
-            'refresh_token': self.refresh_token,
-            'profile_id': self.profile_id
-        }
-
-    def get_refresh_token(self):
-        params = {'profile_id': self.profile_id}
-        heads = self.heads
-        url_path = self.re_url_path
-        tem = request(url_path=url_path, head=heads, params=params)
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            out = _[0].get('refresh_token')
-        else:
-            out = None
-        return out
-
-    def get_spkeywords_data(self):
-        tem = SPClient(**self.AWS_CREDENTIALS)
-        list_spkeywords = tem.iter_keywords(**{"includeExtendedDataFields": True})
-        df_spkeywords = pd.json_normalize(list(list_spkeywords))
-        return df_spkeywords
-
-    def dataconvert(self):
-        df = self.get_spkeywords_data()
-        df['profile_id'] = self.profile_id
-        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
-            '%Y-%m-%d %H:%M:%S')
-        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
-            '%Y-%m-%d %H:%M:%S')
-        col = ['keywordId', 'nativeLanguageKeyword', 'nativeLanguageLocale', 'campaignId', 'matchType', 'state', 'bid',
-               'adGroupId', 'keywordText', 'extendedData.creationDateTime',
-               'extendedData.lastUpdateDateTime', 'extendedData.servingStatus', 'extendedData.servingStatusDetails']
-        old = pd.DataFrame(data=[], columns=col)
-        tem = pd.concat([old, df]).reset_index()
-        tem.drop(columns='index', inplace=True)
-        tem.columns = [i.replace(".", "_") for i in tem.columns]
-        tem.rename(columns={
-            'adGroupId': 'adGroup',
-            'campaignId': 'campaign',
-            'extendedData_creationDateTime': 'creationDateTime',
-            'extendedData_lastUpdateDateTime': 'lastUpdateDateTime',
-            'extendedData_servingStatus': 'servingStatus',
-            'extendedData_servingStatusDetails': 'servingStatusDetails'
-        }, inplace=True)
-        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
-        return json_data
-
-    def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
-
-
-if __name__ == '__main__':
-    a = SpKeyword(profile_id="3006125408623189")
-    out = a.updata_create()
-    # out = a.dataconvert()
-    print(out)

sync_amz_data/tasks/datainsert/alldata_insert.py

@@ -1,36 +0,0 @@
-from mysql_datainsert_portfolios import Portfolios
-from SP.mysql_datainsert_spcampaign import SpCampaign
-from SP.mysql_datainsert_spgroup import SpGroup
-from SP.mysql_datainsert_spkeyword import SpKeyword
-from SP.mysql_datainsert_sp_budget_recommendation import SpBudgetRecommendation
-from SP.mysql_datainsert_spads import SpAds
-from SP.mysql_datainsert_sp_targetsbid_recommendations import SpTargetsBidRecommendations
-
-
-pf = Portfolios("3006125408623189")
-pfo = pf.updata_create()
-print("Portfolios", pfo)
-
-spc = SpCampaign(profile_id="3006125408623189")
-spco = spc.updata_create()
-print("SpCampaign", spco)
-
-spg = SpGroup(profile_id="3006125408623189")
-spgo = spg.updata_create()
-print("SpGroup", spgo)
-
-spk = SpKeyword(profile_id="3006125408623189")
-spko = spk.updata_create()
-print("SpKeyword", spko)
-
-spa = SpAds(profile_id="3006125408623189")
-spao = spa.updata_create()
-print("SpAds", spao)
-
-spbr = SpBudgetRecommendation(profile_id="3006125408623189")
-spbro = spbr.updata_create()
-print("SpBudgetRecommendation", spbro)
-
-sptb = SpTargetsBidRecommendations(profile_id="3006125408623189")
-sptbo = sptb.updata_create()
-print("SpTargetsBidRecommendations", sptbo)

sync_amz_data/tasks/datainsert/mysql_datainsert_portfolios.py

@@ -1,90 +0,0 @@
-import requests
-from urllib.parse import urljoin
-from sync_amz_data.public.amz_ad_client import Account
-from sync_amz_data.settings import AWS_LWA_CLIENT
-import pandas as pd
-import json
-import time
-
-class RateLimitError(Exception):
-    def __init__(self, retry_after: str = None):
-        self.retry_after = retry_after
-
-
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
-    resp = requests.session().request(
-        method=method,
-        url=urljoin(ADS, url_path),
-        headers=head,
-        params=params,
-        json=body,
-    )
-    if resp.status_code == 429:
-        raise RateLimitError(resp.headers.get("Retry-After"))
-    if resp.status_code >= 400:
-        raise Exception(resp.text)
-
-    return resp.json()
-
-class Portfolios:
-    def __init__(self, profile_id):
-        self.profile_id = profile_id
-        self.re_url_path = "api/ad_manage/profiles/"
-        self.upcreate_url_path = "api/ad_manage/portfolios/updata/"
-        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
-        self.refresh_token = self.get_refresh_token()
-        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
-        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
-        self.AWS_CREDENTIALS = {
-            'lwa_client_id': self.lwa_client_id,
-            'lwa_client_secret': self.lwa_client_secret,
-            'refresh_token': self.refresh_token,
-            'profile_id': self.profile_id
-        }
-
-    def get_refresh_token(self):
-        params = {'profile_id': self.profile_id}
-        heads = self.heads
-        url_path = self.re_url_path
-        tem = request(url_path=url_path, head=heads, params=params)
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            data = _[0].get('refresh_token')
-        else:
-            data = None
-        return data
-
-    def get_portfolios_data(self):
-        tem = Account(**self.AWS_CREDENTIALS)
-        list_portfolio = tem.get_portfolios()
-        df_portfolio = pd.json_normalize(list_portfolio)
-        return df_portfolio
-
-    def dataconvert(self):
-        df = self.get_portfolios_data()
-        df['profile_id'] = self.profile_id
-        df['creationDate'] = pd.to_datetime(df['creationDate'], unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
-        df['lastUpdatedDate'] = pd.to_datetime(df['lastUpdatedDate'], unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
-        col = ['profile_id', 'portfolioId', 'name', 'inBudget', 'state', 'creationDate',
-               'lastUpdatedDate', 'servingStatus', 'budget.amount',
-               'budget.currencyCode', 'budget.policy', 'budget.startDate',
-               'budget.endDate']
-        old = pd.DataFrame(data=[], columns=col)
-        tem = pd.concat([old, df])
-        tem.columns = [i.replace(".", "_") for i in tem.columns]
-        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
-        return json_data
-
-    def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
-
-if __name__ == '__main__':
-    a = Portfolios("3006125408623189")
-    out = a.updata_create()
-    # out = a.dataconvert()
-    print(out)

sync_amz_data/tasks/datainsert/mysql_datainsert_sdcampaign.py

@@ -1,104 +0,0 @@
-import requests
-from urllib.parse import urljoin
-from sync_amz_data.public.amz_ad_client import SDClient
-from sync_amz_data.settings import AWS_LWA_CLIENT
-import pandas as pd
-import json
-
-
-class RateLimitError(Exception):
-    def __init__(self, retry_after: str = None):
-        self.retry_after = retry_after
-
-
-def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
-    ADS = "http://192.168.1.23:8001/"
-    resp = requests.session().request(
-        method=method,
-        url=urljoin(ADS, url_path),
-        headers=head,
-        params=params,
-        json=body,
-    )
-    if resp.status_code == 429:
-        raise RateLimitError(resp.headers.get("Retry-After"))
-    if resp.status_code >= 400:
-        raise Exception(resp.text)
-    return resp.json()
-
-class SpCampaign:
-    def __init__(self, profile_id, portfolioId=None):
-        self.profile_id = profile_id
-        self.portfolioId = portfolioId
-        self.re_url_path = "api/ad_manage/profiles/"
-        self.upcreate_url_path = "api/ad_manage/sdcampaigns/updata/"
-        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
-        self.refresh_token = self.get_refresh_token()
-        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
-        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
-        self.AWS_CREDENTIALS = {
-            'lwa_client_id': self.lwa_client_id,
-            'lwa_client_secret': self.lwa_client_secret,
-            'refresh_token': self.refresh_token,
-            'profile_id': self.profile_id
-        }
-
-    def get_refresh_token(self):
-        params = {'profile_id': self.profile_id}
-        heads = self.heads
-        url_path = self.re_url_path
-        tem = request(url_path=url_path, head=heads, params=params)
-        if tem.get('data') is not None:
-            _ = tem.get('data')
-            out = _[0].get('refresh_token')
-        else:
-            out = None
-        return out
-
-    def get_sdcampaign_data(self):
-        tem = SDClient(**self.AWS_CREDENTIALS)
-        list_campaigns = tem.get_campaigns_extended()
-        df_campaigns = pd.json_normalize(list_campaigns)
-        return df_campaigns
-
-    def dataconvert(self):
-        df = self.get_sdcampaign_data()
-        df['profile_id'] = self.profile_id
-        df['creationDate'] = pd.to_datetime(df['creationDate'], unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
-        df['lastUpdatedDate'] = pd.to_datetime(df['lastUpdatedDate'], unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
-        df['portfolioId'] = df['portfolioId'].apply(lambda x: str(int(x)) if not pd.isna(x) else x)
-        df['startDate'] = pd.to_datetime(df['startDate'], format='%Y%m%d').dt.strftime('%Y-%m-%d')
-        if 'endDate' in df.columns:
-            df['endDate'] = pd.to_datetime(df['endDate'], format='%Y%m%d').dt.strftime('%Y-%m-%d')
-        col = ['campaignId', 'tactic', 'name', 'startDate', 'budget', 'budgetType',
-               'endDate', 'state', 'portfolioId', 'deliveryProfile', 'costType',
-               'creationDate', 'lastUpdatedDate', 'profile_id', 'servingStatus',
-               'ruleBasedBudget.isProcessing', 'ruleBasedBudget.applicableRuleName',
-               'ruleBasedBudget.value', 'ruleBasedBudget.applicableRuleId']
-        old = pd.DataFrame(data=[], columns=col)
-        tem = pd.concat([old, df]).reset_index()
-        tem.drop(columns='index', inplace=True)
-        tem.columns = [i.replace(".", "_") for i in tem.columns]
-        tem.rename(columns={
-            'name': 'campaignName',
-            'portfolioId': 'portfolio',
-            'ruleBasedBudget_isProcessing': 'rbb_isProcessing',
-            'ruleBasedBudget_applicableRuleName': 'rbb_appRuleName',
-            'ruleBasedBudget_value': 'rbb_value',
-            'ruleBasedBudget_applicableRuleId': 'rbb_appRuleId'}, inplace=True)
-        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
-        return json_data
-
-    def updata_create(self):
-        body = self.dataconvert()
-        heads = self.heads
-        url_path = self.upcreate_url_path
-        tem = request(url_path=url_path, head=heads, body=body, method="POST")
-        return tem
-
-
-if __name__ == '__main__':
-    a = SpCampaign(profile_id="3006125408623189")
-    out = a.updata_create()
-    # out = a.dataconvert()
-    print(out)

sync_amz_data/tasks/sb.py

@@ -1,16 +0,0 @@
-from sync_amz_data.public import SBClient
-from sync_amz_data.tasks._base import BaseTask
-
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-class SBTask(BaseTask):
-    AmzAdClientClass = SBClient
-
-    def change_campaigns(self, data: dict):
-        return data
-
-    def change_groups(self, data: dict):
-        return data

sync_amz_data/tasks/sd.py

@@ -1,10 +0,0 @@
-from sync_amz_data.public import SDClient
-from sync_amz_data.tasks._base import BaseTask
-
-import logging
-
-logger = logging.getLogger(__name__)
-
-
-class SDTask(BaseTask):
-    AmzAdClientClass = SDClient

sync_amz_data/tasks/sp.py

@@ -1,9 +0,0 @@
-from sync_amz_data.public import SPClient
-from sync_amz_data.tasks._base import BaseTask
-
-
-class SpTask(BaseTask):
-    AmzAdClientClass = SPClient
-
-    def change_campaigns(self, data: dict):
-        return data