|
|
@@ -11,7 +11,7 @@ pd.set_option('expand_frame_repr', False)
|
|
|
from datetime import datetime, timezone, timedelta
|
|
|
import clickhouse_connect
|
|
|
import pytz
|
|
|
-
|
|
|
+from typing import Literal
|
|
|
|
|
|
class Common_ETLMethod(BaseClient):
|
|
|
def timeZone(self):
|
|
|
@@ -69,11 +69,26 @@ class Common_ETLMethod(BaseClient):
|
|
|
df[segment + str("_type")] = df[segment].str.extract(r"'type':\s{0,1}'(.+?)',")
|
|
|
df[segment + str("_value")] = df[segment].str.extract(r"'value':\s{0,1}[\'\[\{](.+)'")
|
|
|
df[segment + str("_value")] = df[segment + str("_value")].map(
|
|
|
- lambda x: x if pd.isna(x) or "," not in x else "[{" + x + "'}]")
|
|
|
+ lambda x: x if pd.isna(x) or "," not in x else "[{" + x + "'}]").str.replace("{{","{")
|
|
|
df.replace(['nan', 'Nan', 'NaN'], np.nan, inplace=True)
|
|
|
df.drop(columns=[segment], inplace=True)
|
|
|
return df
|
|
|
|
|
|
+ def get_keyOvalue(self,expressions, result: Literal['value', 'type']):
|
|
|
+ if expressions not in [None,pd.NA,pd.NaT]:
|
|
|
+ if len(expressions) > 1:
|
|
|
+ return [i[result] for i in expressions]
|
|
|
+ if len(expressions) == 1:
|
|
|
+ sub_ = expressions[0][result]
|
|
|
+ if type(sub_) == str:
|
|
|
+ return sub_
|
|
|
+ elif type(sub_) == list:
|
|
|
+ return [i[result] for i in sub_]
|
|
|
+ else:
|
|
|
+ return expressions
|
|
|
+ else:
|
|
|
+ return '-'
|
|
|
+
|
|
|
def type_trans(self, df_report, columns: list, timeZone_: str, extra_columns: list = []):
|
|
|
# 添加字段
|
|
|
df_report['profileId'] = self.profile_id
|
|
|
@@ -1442,6 +1457,7 @@ if __name__ == '__main__':
|
|
|
ac_etl = SD_ETL(**AWS_CREDENTIALS)
|
|
|
ls = ac_etl.targets_ETL()
|
|
|
print(ls)
|
|
|
+ print(ls.tail(20).to_excel('obse1.xlsx'))
|
|
|
# ac_etl.reportV2_campaignsRecord_t2_ETL(conn)
|
|
|
# ac_etl.reportV2_campaignsRecord_t3_ETL(conn)
|
|
|
# ac_etl.reportV2_adGroupsRecord_t2_ETL(conn)
|
huangyifan