add it

sync_amz_data/tasks/__init__.py

@@ -0,0 +1,2 @@
+from .sb import SBTask
+

sync_amz_data/tasks/_base.py

@@ -0,0 +1,65 @@
+from functools import lru_cache
+import logging
+from typing import List
+
+from sync_amz_data.public import BaseClient, asj_api
+from sync_amz_data.settings import AWS_LWA_CLIENT, DATA_PATH
+
+logger = logging.getLogger(__name__)
+ASJ_URL_BASE = "http://127.0.0.1:8000/api/ad_manage/"
+
+
+@lru_cache(maxsize=100)
+def query_shop_info(profile_id: str) -> dict:
+    # todo 临时
+    if profile_id == "3006125408623189":
+        return {
+            "profile_id": profile_id,
+            "id": 1,
+            "shop_name": "ZosiDirect",
+            "region": "NA",
+            "access_token": "Atza|IwEBION12cAeJLW1CrhirYbH8ianuTfh1JQtC2nbnZjWcCk_J0v9XVF00Pm7AYzZPHXAuFgdul0vuQt4XUAcwVdzQF9AzDfk5wKAUXe9fuGZhb0nq0mdJxg08u2BR_rUggxKWSd1sg6OW7szIEq8xQzok9hcr-Ai-WupyaA-CCznOt7STmyEZltNsK8VuJb7ySxTlxwf-DbuX2Tn9JdsEta7DwQIsHcYv2QwsfLYGnk2LcLUUXG-6TtwCWFGHMxHfoScOvN92hOHiPl3CsdTs5RmKO5eVFdf0XUu8OU5Z9icnjuP1tYBy7_e9s2oTL9fVLLfH_ATUJplPsfm1MhtW6ioX9IXxfJSmOJ0ntEv45ndb9t-wHE_vLukuy_4jwmy_50NU_TOU_9pbFJLQhVyAB0f4HVcb5fgPn--feAX89ANhRJnIn5zibVk_rY_rte7Xu7JMJNNW41PoCUvfVdxjnorhGTIGh2u2JHQPqdww2xLcZ93SQ",
+            "status": 1,
+            "refresh_token": "Atzr|IwEBIL4ur8kbcwRyxVu_srprAAoTYzujnBvA6jU-0SMxkRgOhGjYJSUNGKvw24EQwJa1jG5RM76mQD2P22AKSq8qSD94LddoXGdKDO74eQVYl0RhuqOMFqdrEZpp1p4bIR6_N8VeSJDHr7UCuo8FiabkSHrkq7tsNvRP-yI-bnpQv4EayPBh7YwHVX3hYdRbhxaBvgJENgCuiEPb35Q2-Z6w6ujjiKUAK2VSbCFpENlEfcHNsjDeY7RCvFlwlCoHj1IeiNIaFTE9yXFu3aEWlExe3LzHv6PZyunEi88QJSXKSh56Um0e0eEg05rMv-VBM83cAqc5POmZnTP1vUdZO8fQv3NFLZ-xU6e1WQVxVPi5Cyqk4jYhGf1Y9t98N654y0tVvw74qNIsTrB-8bGS0Uhfe24oBEWmzObvBY3zhtT1d42myGUJv4pMTU6yPoS83zhPKm3LbUDEpBA1hvvc_09jHk7vUEAuFB-UAZzlht2C1yklzQ",
+            "update_time": 1688351991,
+            "token_expires_time": 1688355491,
+            "create_time": 1683702488
+        }
+    return {}
+
+
+class BaseTask:
+    AmzAdClientClass = BaseClient
+
+    def __init__(self, profile_id: str):
+        self.shop_info = query_shop_info(profile_id)
+        self.ad_cil = self.AmzAdClientClass(
+            profile_id=profile_id,
+            refresh_token=self.shop_info["refresh_token"],
+            data_path=DATA_PATH, **AWS_LWA_CLIENT)
+
+    def do(self, task_info: dict):
+        record = task_info["record"]
+        iter_records = getattr(self.ad_cil, f"iter_{record}", None)
+        change_func = getattr(self, f"change_{record}", None)
+        ad_api_params = task_info.get("params")
+        if ad_api_params is None:
+            records_iterator = iter_records()
+        else:
+            records_iterator = iter_records(ad_api_params)
+        for data in records_iterator:
+            if change_func:
+                data = change_func(data)
+            self.to_mysql(record, data)
+            logger.info(data)
+
+    def batch_do(self, task_info: dict):
+        """
+        适用于使用pandas等进行批处理
+        @param task_info:
+        @return:
+        """
+        pass
+
+    def to_mysql(self, record: str, data: [dict, List[dict]]):
+        asj_api.create(f"{ASJ_URL_BASE}{record}/", data)

sync_amz_data/tasks/account.py

@@ -0,0 +1,19 @@
+from ._base import BaseTask
+from sync_amz_data.public import AccountClient
+from sync_amz_data.tools import timestamp2utc_dt
+
+
+class AccountTask(BaseTask):
+    AmzAdClientClass = AccountClient
+
+    def change_portfolios(self, data: dict):
+        data["shop"] = self.shop_info["profile_id"]
+        for key in ["lastUpdatedDate", "creationDate"]:
+            if key in data:
+                data[key] = timestamp2utc_dt(data[key])
+
+        budget: dict = data.pop("budget", None)
+        if budget:
+            for key, val in budget.items():
+                data[f"budget_{key}"] = val
+        return data

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbads.py

@@ -0,0 +1,106 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SBClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SbAds:
+    def __init__(self, profile_id, campaignId: list = None):
+        self.profile_id = profile_id
+        self.portfolioId = campaignId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/sbads/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_sbads_data(self):
+        tem = SBClient(**self.AWS_CREDENTIALS)
+        list_ads = tem.iter_ads(**{"includeExtendedDataFields": True})
+        df_ads = pd.json_normalize(list(list_ads))
+        return df_ads
+
+    def dataconvert(self):
+        df = self.get_sbads_data()
+        df['extendedData.creationDate'] = pd.to_datetime(df['extendedData.creationDate'],
+                                                             unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
+        df['extendedData.lastUpdateDate'] = pd.to_datetime(df['extendedData.lastUpdateDate'],
+                                                               unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
+        col = ['adId', 'campaignId', 'landingPage.asins', 'landingPage.pageType', 'landingPage.url', 'name', 'state',
+               'adGroupId', 'creative.asins', 'creative.type', 'extendedData.creationDate',
+               'extendedData.lastUpdateDate', 'extendedData.servingStatus', 'extendedData.servingStatusDetails']
+        for i in col:
+            if i not in df.columns:
+                df['{}'.format(i)] = None
+        df_new = df[col].copy()
+        old = pd.DataFrame(data=[], columns=col)
+        tem = pd.concat([old, df_new]).reset_index()
+        tem.drop(columns='index', inplace=True)
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem.rename(columns={
+            'adGroupId': 'adGroup',
+            'campaignId': 'campaign',
+            'extendedData_creationDate': 'creationDateTime',
+            'extendedData_lastUpdateDate': 'lastUpdateDateTime',
+            'extendedData_servingStatus': 'servingStatus',
+            'extendedData_servingStatusDetails': 'servingStatusDetails',
+            'name': 'adName',
+        }, inplace=True)
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SbAds(profile_id="3006125408623189")
+    out = a.updata_create()
+    # out = a.dataconvert()
+    print(out)

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbcampaign.py

@@ -0,0 +1,123 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SBClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SbCampaign:
+    def __init__(self, profile_id, portfolioId: list = None):
+        self.profile_id = profile_id
+        self.portfolioId = portfolioId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/sbcampaigns/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_sbcampaign_data(self):
+        tem = SBClient(**self.AWS_CREDENTIALS)
+        if self.portfolioId is None:
+            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True,
+                                                   "stateFilter": {
+                                                       "include": ["ARCHIVED", "PAUSED", "ENABLED"]
+                                                   }})
+        else:
+            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True,
+                                                   "portfolioIdFilter": {"include": self.portfolioId},
+                                                   "stateFilter": {
+                                                       "include": ["ARCHIVED", "PAUSED", "ENABLED"]
+                                                   }})
+        df_campaigns = pd.json_normalize(list(list_campaigns))
+        return df_campaigns
+
+    def dataconvert(self):
+        df = self.get_sbcampaign_data()
+        df['profile_id'] = self.profile_id
+        df['extendedData.creationDate'] = pd.to_datetime(df['extendedData.creationDate'], unit='ms').dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        df['extendedData.lastUpdateDate'] = pd.to_datetime(df['extendedData.lastUpdateDate'], unit='ms').dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        col = ['budgetType', 'ruleBasedBudget.isProcessing', 'ruleBasedBudget.applicableRuleName',
+               'ruleBasedBudget.value', 'ruleBasedBudget.applicableRuleId', 'brandEntityId',
+               'isMultiAdGroupsEnabled', 'goal', 'bidding.bidOptimization', 'bidding.bidAdjustmentsByShopperSegment',
+               'bidding.bidAdjustmentsByPlacement', 'bidding.bidOptimizationStrategy', 'endDate',
+               'campaignId', 'productLocation', 'portfolioId', 'costType', 'smartDefault', 'name', 'state',
+               'startDate', 'budget', 'extendedData.servingStatus', 'extendedData.lastUpdateDate',
+               'extendedData.creationDate', 'extendedData.servingStatusDetails']
+        old = pd.DataFrame(data=[], columns=col)
+        tem = pd.concat([old, df]).reset_index()
+        tem.drop(columns='index', inplace=True)
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem.rename(columns={
+            'name': 'campaignName',
+            'portfolioId': 'portfolio',
+            'ruleBasedBudget_isProcessing': 'rbb_isProcessing',
+            'ruleBasedBudget_applicableRuleName': 'rbb_appRuleName',
+            'ruleBasedBudget_value': 'rbb_value',
+            'ruleBasedBudget_applicableRuleId': 'rbb_appRuleId',
+            'bidding_bidOptimization': 'bidOptimization',
+            'bidding_bidAdjustmentsByShopperSegment': 'bidShopperSegment',
+            'bidding_bidAdjustmentsByPlacement': 'bidPlacement',
+            'bidding_bidOptimizationStrategy': 'bidOptimizationStrategy',
+            'extendedData_lastUpdateDate': 'lastUpdateDate',
+            'extendedData_servingStatus': 'servingStatus',
+            'extendedData_servingStatusDetails': 'servingStatusDetails',
+            'extendedData_creationDate': 'creationDate'}, inplace=True)
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SbCampaign(profile_id="3006125408623189")
+    out = a.updata_create()
+    # out = a.dataconvert()
+    print(out)

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbgroup.py

@@ -0,0 +1,104 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SBClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SbGroup:
+    def __init__(self, profile_id, campaignId: list = None):
+        self.profile_id = profile_id
+        self.portfolioId = campaignId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/sbgroups/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_spgroup_data(self):
+        tem = SBClient(**self.AWS_CREDENTIALS)
+        list_group = tem.iter_adGroups(**{"includeExtendedDataFields": True,
+                                          "stateFilter": {
+                                              "include": ["ARCHIVED", "PAUSED", "ENABLED"]
+                                          }})
+        df_group = pd.json_normalize(list(list_group))
+        return df_group
+    #----------------------------
+
+    def dataconvert(self):
+        df = self.get_spgroup_data()
+        df['extendedData.creationDate'] = pd.to_datetime(
+            df['extendedData.creationDate'], unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
+        df['extendedData.lastUpdateDate'] = pd.to_datetime(
+            df['extendedData.lastUpdateDate'], unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
+        col = ['adGroupId', 'campaignId', 'name', 'state', 'extendedData.creationDate',
+               'extendedData.lastUpdateDate', 'extendedData.servingStatus', 'extendedData.servingStatusDetails']
+        old = pd.DataFrame(data=[], columns=col)
+        tem = pd.concat([old, df]).reset_index()
+        tem.drop(columns='index', inplace=True)
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem.rename(columns={
+            'name': 'adGroupName',
+            'campaignId': 'campaign',
+            'extendedData_creationDate': 'creationDateTime',
+            'extendedData_lastUpdateDate': 'lastUpdateDateTime',
+            'extendedData_servingStatus': 'servingStatus',
+            'extendedData_servingStatusDetails': 'servingStatusDetails'
+        }, inplace=True)
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SbGroup(profile_id="3006125408623189")
+    out = a.updata_create()
+    # out = a.dataconvert()
+    print(out)

sync_amz_data/tasks/datainsert/SB/mysql_datainsert_sbkeyword_v3.py

@@ -0,0 +1,97 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SBClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SpKeyword:
+    def __init__(self, profile_id, campaignId: list = None):
+        self.profile_id = profile_id
+        self.portfolioId = campaignId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/sbkeywords/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_sbkeywords_data(self):
+        tem = SBClient(**self.AWS_CREDENTIALS)
+        _ = tem.iter_keywords()
+        list_sbkeywords = [row for i in _ for row in i]
+        df_sbkeywords = pd.json_normalize(list(list_sbkeywords))
+        return df_sbkeywords
+
+    def dataconvert(self):
+        df = self.get_sbkeywords_data()
+        col = ['keywordId', 'adGroupId', 'campaignId', 'keywordText',
+               'nativeLanguageKeyword', 'matchType', 'state', 'bid']
+        old = pd.DataFrame(data=[], columns=col)
+        for i in col:
+            if i not in df.columns:
+                df['{}'.format(i)] = None
+        df_new = df[col].copy()
+        tem = pd.concat([old, df_new]).reset_index()
+        tem.drop(columns='index', inplace=True)
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem.rename(columns={
+            'adGroupId': 'adGroup',
+            'campaignId': 'campaign',
+        }, inplace=True)
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SpKeyword(profile_id="3006125408623189")
+    out = a.updata_create()
+    # out = a.dataconvert()
+    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sp_budget_recommendation.py

@@ -0,0 +1,128 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SPClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SpBudgetRecommendation:
+    def __init__(self, profile_id, portfolioId: list = None):
+        self.profile_id = profile_id
+        self.portfolioId = portfolioId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.camid_url_path = "api/ad_manage/spcampaigns/?updata=1&limit=9999"
+        self.upcreate_url_path = "api/ad_manage/spbudgetrecommendation/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_campaignid(self):
+        heads = self.heads
+        url_path = self.camid_url_path
+        tem = request(url_path=url_path, head=heads)
+        out = []
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            for i in _:
+                out.append(i['campaignId'])
+        return out
+
+    def get_budgetrecommendation_data(self):
+        tem = SPClient(**self.AWS_CREDENTIALS)
+        camid_list = self.get_campaignid()
+        list_tem = tem.iter_budgetrecommendation(camid_list)
+        df_spbudgetrecommendation = pd.json_normalize(list(list_tem))
+        return df_spbudgetrecommendation
+
+    def dataconvert(self):
+        df = self.get_budgetrecommendation_data()
+        df.drop(columns='index', inplace=True)
+        df['sevenDaysMissedOpportunities.startDate'] = pd.to_datetime(
+            df['sevenDaysMissedOpportunities.startDate'], format='%Y%m%d').dt.strftime(
+            '%Y-%m-%d')
+        df['sevenDaysMissedOpportunities.endDate'] = pd.to_datetime(
+            df['sevenDaysMissedOpportunities.endDate'], format='%Y%m%d').dt.strftime(
+            '%Y-%m-%d')
+        col = ['campaignId', 'suggestedBudget', 'budgetRuleRecommendation',
+               'sevenDaysMissedOpportunities.startDate',
+               'sevenDaysMissedOpportunities.endDate',
+               'sevenDaysMissedOpportunities.percentTimeInBudget',
+               'sevenDaysMissedOpportunities.estimatedMissedImpressionsLower',
+               'sevenDaysMissedOpportunities.estimatedMissedImpressionsUpper',
+               'sevenDaysMissedOpportunities.estimatedMissedClicksLower',
+               'sevenDaysMissedOpportunities.estimatedMissedClicksUpper',
+               'sevenDaysMissedOpportunities.estimatedMissedSalesLower',
+               'sevenDaysMissedOpportunities.estimatedMissedSalesUpper']
+        old = pd.DataFrame(data=[], columns=col)
+        tem = pd.concat([old, df]).reset_index()
+        tem.drop(columns='index', inplace=True)
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem.rename(columns={
+            'campaignId': 'campaign',
+            'sevenDaysMissedOpportunities_startDate': 'startDate',
+            'sevenDaysMissedOpportunities_endDate': 'endDate',
+            'sevenDaysMissedOpportunities_percentTimeInBudget': 'percentTimeInBudget',
+            'sevenDaysMissedOpportunities_estimatedMissedImpressionsLower': 'MissedImpressionsLower',
+            'sevenDaysMissedOpportunities_estimatedMissedImpressionsUpper': 'MissedImpressionsUpper',
+            'sevenDaysMissedOpportunities_estimatedMissedClicksLower': 'MissedClicksLower',
+            'sevenDaysMissedOpportunities_estimatedMissedClicksUpper': 'MissedClicksUpper',
+            'sevenDaysMissedOpportunities_estimatedMissedSalesLower': 'MissedSalesLower',
+            'sevenDaysMissedOpportunities_estimatedMissedSalesUpper': 'MissedSalesUpper',
+        }, inplace=True)
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SpBudgetRecommendation(profile_id="3006125408623189")
+    out = a.updata_create()
+    # out = a.dataconvert()
+    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_sp_targetsbid_recommendations.py

@@ -0,0 +1,132 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SPClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+pd.set_option('display.max_columns', None)
+# 显示所有行
+pd.set_option('display.max_rows', None)
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SpTargetsBidRecommendations:
+    def __init__(self, profile_id):
+        self.profile_id = profile_id
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.cgk_url_path = "api/ad_manage/sptbrkeywords/"
+        self.upcreate_url_path = "api/ad_manage/sptargetsbidrecommendation/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_arg(self):
+        heads = self.heads
+        url_path = self.cgk_url_path
+        data = []
+        page = 1
+        params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
+        tem = request(url_path=url_path, head=heads, params=params)
+        data.extend(tem.get('data'))
+        while tem.get('is_next') is True:
+            page += 1
+            params = {'profile_id': self.profile_id, 'limit': 999, 'page': page}
+            tem = request(url_path=url_path, head=heads, params=params)
+            data.extend(tem.get('data'))
+        _ = pd.json_normalize(data)
+        df = _.copy()
+        df.rename(columns={'keywordText': 'value', 'matchType': 'type'}, inplace=True)
+        df['targetingExpressions'] = df[['value', 'type', 'keywordId']].apply(lambda x: x.to_dict(), axis=1)
+        df_grouped = df.groupby(['campaignId', 'adGroupId']).agg({'targetingExpressions': list}).reset_index()
+        return df_grouped
+
+    def get_sptargetsbidrecommendation_data(self):
+        tem = SPClient(**self.AWS_CREDENTIALS)
+        df_arg = self.get_arg()
+        data_json = df_arg.to_json(orient='records')
+        list_arg = json.loads(data_json)
+        out_df = pd.DataFrame()
+        for i in list_arg:
+            k_id_text_df = pd.DataFrame.from_dict(i['targetingExpressions'])
+            list_sptbr = tem.iter_adgroup_bidrecommendation(**i)
+            list_outdata = list(list_sptbr)
+            if len(list_outdata) > 0:
+                out_data = []
+                for j in list_outdata:
+                    if j.get('theme') == "CONVERSION_OPPORTUNITIES":
+                        data = j.get('bidRecommendationsForTargetingExpressions')
+                        out_data.extend(data)
+                temtest = pd.json_normalize(out_data)
+                temtest.rename(columns={'targetingExpression.value': 'value'}, inplace=True)
+                temtest.rename(columns={'targetingExpression.type': 'type'}, inplace=True)
+                df_tem = pd.merge(left=temtest, right=k_id_text_df, on=['value', 'type'], how='left')
+                out_df = pd.concat([out_df, df_tem])
+        out_df['suggestedBid'] = out_df.apply(
+            lambda row: row['bidValues'][1]['suggestedBid'] if len(row['bidValues']) > 0 else None,
+            axis=1)
+        out_df['suggestedBid_lower'] = out_df.apply(
+            lambda row: row['bidValues'][0]['suggestedBid'] if len(row['bidValues']) > 0 else None,
+            axis=1)
+        out_df['suggestedBid_upper'] = out_df.apply(
+            lambda row: row['bidValues'][2]['suggestedBid'] if len(row['bidValues']) > 0 else None,
+            axis=1)
+        out_df.drop(labels='bidValues', inplace=True, axis=1)
+        out_df.drop(labels='value', inplace=True, axis=1)
+        out_df.rename(columns={'type': 'targetingExpression_type',
+                               'keywordId': 'keyword'}, inplace=True)
+        json_data = json.loads(out_df.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.get_sptargetsbidrecommendation_data()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SpTargetsBidRecommendations(profile_id="3006125408623189")
+    # out = a.get_sptargetsbidrecommendation_data()
+    out = a.updata_create()
+    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spads.py

@@ -0,0 +1,100 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SPClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SpAds:
+    def __init__(self, profile_id, campaignId: list = None):
+        self.profile_id = profile_id
+        self.portfolioId = campaignId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/spads/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_spads_data(self):
+        tem = SPClient(**self.AWS_CREDENTIALS)
+        list_ads = tem.iter_ads(**{"includeExtendedDataFields": True})
+        df_ads = pd.json_normalize(list(list_ads))
+        return df_ads
+
+    def dataconvert(self):
+        df = self.get_spads_data()
+        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        col = ['adId', 'campaignId', 'customText', 'asin', 'state', 'sku', 'adGroupId', 'extendedData.creationDateTime',
+               'extendedData.lastUpdateDateTime', 'extendedData.servingStatus', 'extendedData.servingStatusDetails']
+        old = pd.DataFrame(data=[], columns=col)
+        tem = pd.concat([old, df]).reset_index()
+        tem.drop(columns='index', inplace=True)
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem.rename(columns={
+            'adGroupId': 'adGroup',
+            'campaignId': 'campaign',
+            'extendedData_creationDateTime': 'creationDateTime',
+            'extendedData_lastUpdateDateTime': 'lastUpdateDateTime',
+            'extendedData_servingStatus': 'servingStatus',
+            'extendedData_servingStatusDetails': 'servingStatusDetails'
+        }, inplace=True)
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SpAds(profile_id="3006125408623189")
+    out = a.updata_create()
+    # out = a.dataconvert()
+    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spcampaign.py

@@ -0,0 +1,114 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SPClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SpCampaign:
+    def __init__(self, profile_id, portfolioId: list = None):
+        self.profile_id = profile_id
+        self.portfolioId = portfolioId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/spcampaigns/updata/?updata=1"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_spcampaign_data(self):
+        tem = SPClient(**self.AWS_CREDENTIALS)
+        if self.portfolioId is None:
+            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True})
+        else:
+            list_campaigns = tem.iter_campaigns(**{"includeExtendedDataFields": True,
+                                                   "portfolioIdFilter": {"include": self.portfolioId}
+                                                   })
+        df_campaigns = pd.json_normalize(list(list_campaigns))
+        return df_campaigns
+
+    def dataconvert(self):
+        df = self.get_spcampaign_data()
+        df['profile_id'] = self.profile_id
+        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        col = ['campaignId', 'endDate', 'name', 'startDate', 'state', 'targetingType',
+               'budget.budget', 'budget.budgetType', 'budget.effectiveBudget',
+               'dynamicBidding.placementBidding', 'extendedData.lastUpdateDateTime',
+               'dynamicBidding.strategy', 'portfolioId', 'profile_id',
+               'extendedData.servingStatus', 'extendedData.servingStatusDetails',
+               'extendedData.creationDateTime']
+        old = pd.DataFrame(data=[], columns=col)
+        tem = pd.concat([old, df]).reset_index()
+        tem.drop(columns='index', inplace=True)
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem.rename(columns={
+            'name': 'campaignName',
+            'portfolioId': 'portfolio',
+            'dynamicBidding_placementBidding': 'dynBidPlacement',
+            'dynamicBidding_strategy': 'dynBidStrategy',
+            'budget_budgetType': 'budgetType',
+            'budget_budget': 'budget',
+            'budget_effectiveBudget': 'effectiveBudget',
+            'extendedData_lastUpdateDateTime': 'lastUpdateDateTime',
+            'extendedData_servingStatus': 'servingStatus',
+            'extendedData_servingStatusDetails': 'servingStatusDetails',
+            'extendedData_creationDateTime': 'creationDateTime'}, inplace=True)
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SpCampaign(profile_id="3006125408623189")
+    out = a.updata_create()
+    # out = a.dataconvert()
+    print(out)

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spgroup.py

@@ -0,0 +1,103 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SPClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SpGroup:
+    def __init__(self, profile_id, campaignId: list = None):
+        self.profile_id = profile_id
+        self.portfolioId = campaignId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/spgroups/updata/?updata=1"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_spgroup_data(self):
+        tem = SPClient(**self.AWS_CREDENTIALS)
+        list_group = tem.iter_adGroups(**{"includeExtendedDataFields": True})
+        df_group = pd.json_normalize(list(list_group))
+        return df_group
+    #----------------------------
+
+    def dataconvert(self):
+        df = self.get_spgroup_data()
+        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        col = ['adGroupId', 'campaignId', 'defaultBid', 'name', 'state', 'extendedData.creationDateTime',
+               'extendedData.lastUpdateDateTime', 'extendedData.servingStatus', 'extendedData.servingStatusDetails']
+        old = pd.DataFrame(data=[], columns=col)
+        tem = pd.concat([old, df]).reset_index()
+        tem.drop(columns='index', inplace=True)
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem.rename(columns={
+            'name': 'adGroupName',
+            'campaignId': 'campaign',
+            'extendedData_creationDateTime': 'creationDateTime',
+            'extendedData_lastUpdateDateTime': 'lastUpdateDateTime',
+            'extendedData_servingStatus': 'servingStatus',
+            'extendedData_servingStatusDetails': 'servingStatusDetails'
+        }, inplace=True)
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SpGroup(profile_id="3006125408623189")
+    out = a.updata_create()
+    # out = a.dataconvert()
+    print(out)
+
+

sync_amz_data/tasks/datainsert/SP/mysql_datainsert_spkeyword.py

@@ -0,0 +1,102 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SPClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+
+class SpKeyword:
+    def __init__(self, profile_id, campaignId: list = None):
+        self.profile_id = profile_id
+        self.campaignId = campaignId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/spkeywords/updata/?updata=1"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_spkeywords_data(self):
+        tem = SPClient(**self.AWS_CREDENTIALS)
+        list_spkeywords = tem.iter_keywords(**{"includeExtendedDataFields": True})
+        df_spkeywords = pd.json_normalize(list(list_spkeywords))
+        return df_spkeywords
+
+    def dataconvert(self):
+        df = self.get_spkeywords_data()
+        df['profile_id'] = self.profile_id
+        df['extendedData.creationDateTime'] = pd.to_datetime(df['extendedData.creationDateTime']).dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        df['extendedData.lastUpdateDateTime'] = pd.to_datetime(df['extendedData.lastUpdateDateTime']).dt.strftime(
+            '%Y-%m-%d %H:%M:%S')
+        col = ['keywordId', 'nativeLanguageKeyword', 'nativeLanguageLocale', 'campaignId', 'matchType', 'state', 'bid',
+               'adGroupId', 'keywordText', 'extendedData.creationDateTime',
+               'extendedData.lastUpdateDateTime', 'extendedData.servingStatus', 'extendedData.servingStatusDetails']
+        old = pd.DataFrame(data=[], columns=col)
+        tem = pd.concat([old, df]).reset_index()
+        tem.drop(columns='index', inplace=True)
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem.rename(columns={
+            'adGroupId': 'adGroup',
+            'campaignId': 'campaign',
+            'extendedData_creationDateTime': 'creationDateTime',
+            'extendedData_lastUpdateDateTime': 'lastUpdateDateTime',
+            'extendedData_servingStatus': 'servingStatus',
+            'extendedData_servingStatusDetails': 'servingStatusDetails'
+        }, inplace=True)
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SpKeyword(profile_id="3006125408623189")
+    out = a.updata_create()
+    # out = a.dataconvert()
+    print(out)

sync_amz_data/tasks/datainsert/alldata_insert.py

@@ -0,0 +1,36 @@
+from mysql_datainsert_portfolios import Portfolios
+from SP.mysql_datainsert_spcampaign import SpCampaign
+from SP.mysql_datainsert_spgroup import SpGroup
+from SP.mysql_datainsert_spkeyword import SpKeyword
+from SP.mysql_datainsert_sp_budget_recommendation import SpBudgetRecommendation
+from SP.mysql_datainsert_spads import SpAds
+from SP.mysql_datainsert_sp_targetsbid_recommendations import SpTargetsBidRecommendations
+
+
+pf = Portfolios("3006125408623189")
+pfo = pf.updata_create()
+print("Portfolios", pfo)
+
+spc = SpCampaign(profile_id="3006125408623189")
+spco = spc.updata_create()
+print("SpCampaign", spco)
+
+spg = SpGroup(profile_id="3006125408623189")
+spgo = spg.updata_create()
+print("SpGroup", spgo)
+
+spk = SpKeyword(profile_id="3006125408623189")
+spko = spk.updata_create()
+print("SpKeyword", spko)
+
+spa = SpAds(profile_id="3006125408623189")
+spao = spa.updata_create()
+print("SpAds", spao)
+
+spbr = SpBudgetRecommendation(profile_id="3006125408623189")
+spbro = spbr.updata_create()
+print("SpBudgetRecommendation", spbro)
+
+sptb = SpTargetsBidRecommendations(profile_id="3006125408623189")
+sptbo = sptb.updata_create()
+print("SpTargetsBidRecommendations", sptbo)

sync_amz_data/tasks/datainsert/mysql_datainsert_portfolios.py

@@ -0,0 +1,90 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import Account
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+import time
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+
+    return resp.json()
+
+class Portfolios:
+    def __init__(self, profile_id):
+        self.profile_id = profile_id
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/portfolios/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            data = _[0].get('refresh_token')
+        else:
+            data = None
+        return data
+
+    def get_portfolios_data(self):
+        tem = Account(**self.AWS_CREDENTIALS)
+        list_portfolio = tem.get_portfolios()
+        df_portfolio = pd.json_normalize(list_portfolio)
+        return df_portfolio
+
+    def dataconvert(self):
+        df = self.get_portfolios_data()
+        df['profile_id'] = self.profile_id
+        df['creationDate'] = pd.to_datetime(df['creationDate'], unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
+        df['lastUpdatedDate'] = pd.to_datetime(df['lastUpdatedDate'], unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
+        col = ['profile_id', 'portfolioId', 'name', 'inBudget', 'state', 'creationDate',
+               'lastUpdatedDate', 'servingStatus', 'budget.amount',
+               'budget.currencyCode', 'budget.policy', 'budget.startDate',
+               'budget.endDate']
+        old = pd.DataFrame(data=[], columns=col)
+        tem = pd.concat([old, df])
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+if __name__ == '__main__':
+    a = Portfolios("3006125408623189")
+    out = a.updata_create()
+    # out = a.dataconvert()
+    print(out)

sync_amz_data/tasks/datainsert/mysql_datainsert_sdcampaign.py

@@ -0,0 +1,104 @@
+import requests
+from urllib.parse import urljoin
+from sync_amz_data.public.amz_ad_client import SDClient
+from sync_amz_data.settings import AWS_LWA_CLIENT
+import pandas as pd
+import json
+
+
+class RateLimitError(Exception):
+    def __init__(self, retry_after: str = None):
+        self.retry_after = retry_after
+
+
+def request(url_path: str, method: str = "GET", head: dict = None, params: dict = None, body: dict = None):
+    ADS = "http://192.168.1.23:8001/"
+    resp = requests.session().request(
+        method=method,
+        url=urljoin(ADS, url_path),
+        headers=head,
+        params=params,
+        json=body,
+    )
+    if resp.status_code == 429:
+        raise RateLimitError(resp.headers.get("Retry-After"))
+    if resp.status_code >= 400:
+        raise Exception(resp.text)
+    return resp.json()
+
+class SpCampaign:
+    def __init__(self, profile_id, portfolioId=None):
+        self.profile_id = profile_id
+        self.portfolioId = portfolioId
+        self.re_url_path = "api/ad_manage/profiles/"
+        self.upcreate_url_path = "api/ad_manage/sdcampaigns/updata/"
+        self.heads = {'X-Token': "da4ab6bc5cbf1dfa"}
+        self.refresh_token = self.get_refresh_token()
+        self.lwa_client_id = AWS_LWA_CLIENT['lwa_client_id']
+        self.lwa_client_secret = AWS_LWA_CLIENT['lwa_client_secret']
+        self.AWS_CREDENTIALS = {
+            'lwa_client_id': self.lwa_client_id,
+            'lwa_client_secret': self.lwa_client_secret,
+            'refresh_token': self.refresh_token,
+            'profile_id': self.profile_id
+        }
+
+    def get_refresh_token(self):
+        params = {'profile_id': self.profile_id}
+        heads = self.heads
+        url_path = self.re_url_path
+        tem = request(url_path=url_path, head=heads, params=params)
+        if tem.get('data') is not None:
+            _ = tem.get('data')
+            out = _[0].get('refresh_token')
+        else:
+            out = None
+        return out
+
+    def get_sdcampaign_data(self):
+        tem = SDClient(**self.AWS_CREDENTIALS)
+        list_campaigns = tem.get_campaigns_extended()
+        df_campaigns = pd.json_normalize(list_campaigns)
+        return df_campaigns
+
+    def dataconvert(self):
+        df = self.get_sdcampaign_data()
+        df['profile_id'] = self.profile_id
+        df['creationDate'] = pd.to_datetime(df['creationDate'], unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
+        df['lastUpdatedDate'] = pd.to_datetime(df['lastUpdatedDate'], unit='ms').dt.strftime('%Y-%m-%d %H:%M:%S')
+        df['portfolioId'] = df['portfolioId'].apply(lambda x: str(int(x)) if not pd.isna(x) else x)
+        df['startDate'] = pd.to_datetime(df['startDate'], format='%Y%m%d').dt.strftime('%Y-%m-%d')
+        if 'endDate' in df.columns:
+            df['endDate'] = pd.to_datetime(df['endDate'], format='%Y%m%d').dt.strftime('%Y-%m-%d')
+        col = ['campaignId', 'tactic', 'name', 'startDate', 'budget', 'budgetType',
+               'endDate', 'state', 'portfolioId', 'deliveryProfile', 'costType',
+               'creationDate', 'lastUpdatedDate', 'profile_id', 'servingStatus',
+               'ruleBasedBudget.isProcessing', 'ruleBasedBudget.applicableRuleName',
+               'ruleBasedBudget.value', 'ruleBasedBudget.applicableRuleId']
+        old = pd.DataFrame(data=[], columns=col)
+        tem = pd.concat([old, df]).reset_index()
+        tem.drop(columns='index', inplace=True)
+        tem.columns = [i.replace(".", "_") for i in tem.columns]
+        tem.rename(columns={
+            'name': 'campaignName',
+            'portfolioId': 'portfolio',
+            'ruleBasedBudget_isProcessing': 'rbb_isProcessing',
+            'ruleBasedBudget_applicableRuleName': 'rbb_appRuleName',
+            'ruleBasedBudget_value': 'rbb_value',
+            'ruleBasedBudget_applicableRuleId': 'rbb_appRuleId'}, inplace=True)
+        json_data = json.loads(tem.to_json(orient='records', force_ascii=False))
+        return json_data
+
+    def updata_create(self):
+        body = self.dataconvert()
+        heads = self.heads
+        url_path = self.upcreate_url_path
+        tem = request(url_path=url_path, head=heads, body=body, method="POST")
+        return tem
+
+
+if __name__ == '__main__':
+    a = SpCampaign(profile_id="3006125408623189")
+    out = a.updata_create()
+    # out = a.dataconvert()
+    print(out)

sync_amz_data/tasks/sb.py

@@ -0,0 +1,16 @@
+from sync_amz_data.public import SBClient
+from sync_amz_data.tasks._base import BaseTask
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class SBTask(BaseTask):
+    AmzAdClientClass = SBClient
+
+    def change_campaigns(self, data: dict):
+        return data
+
+    def change_groups(self, data: dict):
+        return data

sync_amz_data/tasks/sd.py

@@ -0,0 +1,10 @@
+from sync_amz_data.public import SDClient
+from sync_amz_data.tasks._base import BaseTask
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class SDTask(BaseTask):
+    AmzAdClientClass = SDClient

sync_amz_data/tasks/sp.py

@@ -0,0 +1,9 @@
+from sync_amz_data.public import SPClient
+from sync_amz_data.tasks._base import BaseTask
+
+
+class SpTask(BaseTask):
+    AmzAdClientClass = SPClient
+
+    def change_campaigns(self, data: dict):
+        return data