|
|
@@ -89,6 +89,12 @@ class Common_ETLMethod(BaseClient):
|
|
|
else:
|
|
|
return '-'
|
|
|
|
|
|
+ def id_type_trans(self,df):
|
|
|
+ id_columns = [i for i in df.columns if 'id' in i.lower() and 'bid' not in i.lower()]
|
|
|
+ df[id_columns] = df[id_columns].fillna(-1).applymap(lambda x: round(int(x), 0)).astype("string")
|
|
|
+ df[id_columns] = df[id_columns].applymap(lambda x: '' if x == '-1' else x)
|
|
|
+ return df
|
|
|
+
|
|
|
def type_trans(self, df_report, columns: list, timeZone_: str, extra_columns: list = []):
|
|
|
# 添加字段
|
|
|
df_report['profileId'] = self.profile_id
|
|
|
@@ -102,9 +108,7 @@ class Common_ETLMethod(BaseClient):
|
|
|
columns.extend(["timeZone", "profileId"])
|
|
|
|
|
|
# 修改字段类型
|
|
|
- id_columns = [i for i in columns if 'id' in i.lower() and 'bid' not in i.lower()]
|
|
|
- df_report[id_columns] = df_report[id_columns].fillna(-1).applymap(lambda x: round(int(x), 0)).astype("string")
|
|
|
- df_report[id_columns] = df_report[id_columns].applymap(lambda x: '' if x == '-1' else x)
|
|
|
+ df_report = self.id_type_trans(df_report)
|
|
|
df_report['date'] = df_report['date'].astype("datetime64")
|
|
|
|
|
|
# df_report[df_report.select_dtypes('O').columns] = df_report[df_report.select_dtypes('O').columns].astype('string')
|
|
|
@@ -465,6 +469,7 @@ class SB_ETL(SBClient, Common_ETLMethod):
|
|
|
df_targets = pd.json_normalize(list_targets)
|
|
|
# df_targets = self.TZ_Deal(df_targets, ["extendedData.creationDateTime", "extendedData.lastUpdateDateTime"])
|
|
|
# df_targets = self.expression_split(df_targets, "resolvedExpressions")
|
|
|
+ df_targets = self.id_type_trans(df_targets)
|
|
|
df_targets['resolvedExpressions_type'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'type'))
|
|
|
df_targets['resolvedExpressions_value'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'value'))
|
|
|
return self.columnsName_modify(df_targets)
|
|
|
@@ -960,6 +965,7 @@ class SD_ETL(SDClient, Common_ETLMethod):
|
|
|
list_targets = [row for _ in list(self.iter_targets(**param)) for row in _]
|
|
|
df_targets = pd.json_normalize(list_targets)
|
|
|
# df_targets = self.expression_split(df_targets, "resolvedExpression")
|
|
|
+ df_targets = self.id_type_trans(df_targets)
|
|
|
df_targets['resolvedExpressions_type'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'type'))
|
|
|
df_targets['resolvedExpressions_value'] = df_targets['resolvedExpressions'].map(lambda x:self.get_keyOvalue(x,'value'))
|
|
|
return self.columnsName_modify(df_targets)
|
|
|
@@ -1460,7 +1466,7 @@ if __name__ == '__main__':
|
|
|
ac_etl = SB_ETL(**AWS_CREDENTIALS)
|
|
|
ls = ac_etl.targets_ETL()
|
|
|
print(ls)
|
|
|
- print(ls.to_excel('obse111.xlsx'))
|
|
|
+ print(ls.to_excel('obse11.xlsx'))
|
|
|
# ac_etl.reportV2_campaignsRecord_t2_ETL(conn)
|
|
|
# ac_etl.reportV2_campaignsRecord_t3_ETL(conn)
|
|
|
# ac_etl.reportV2_adGroupsRecord_t2_ETL(conn)
|
huangyifan