-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #3 from azuki774/cf-import
add: money-forward-cf
- Loading branch information
Showing
15 changed files
with
482 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,3 +26,7 @@ deployment/browser/ | |
| deployment/*.jpg | ||
|
|
||
| !.gitkeep | ||
|
|
||
| compose.yml | ||
| *-token.env | ||
| tmp | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| FROM golang:1.21.0 as builder | ||
|
|
||
| COPY . /app/ | ||
| WORKDIR /app | ||
|
|
||
| # Go build | ||
| RUN go mod download | ||
| RUN make bin-linux-amd64 | ||
|
|
||
| FROM debian:bookworm-slim as runner | ||
| # Required Packages | ||
| RUN apt-get update && \ | ||
| apt-get install -y curl unzip && \ | ||
| apt-get clean && \ | ||
| rm -rf /var/lib/apt/lists/* | ||
|
|
||
| # AWS Setup | ||
| RUN curl -o /var/tmp/awscli.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && \ | ||
| unzip -d /usr/local/bin/ /var/tmp/awscli.zip | ||
|
|
||
| RUN mkdir -p /usr/local/bin && mkdir -p /data/ | ||
| COPY --from=builder /app/build/bin/myscrapers /usr/local/bin/myscrapers | ||
| COPY --chmod=755 build/moneyforward/main.sh /usr/local/bin/main.sh | ||
| ENTRYPOINT ["/usr/local/bin/main.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| #!/bin/bash | ||
| set -e | ||
| YYYYMM=`date '+%Y%m'` | ||
| YYYYMMDD=`date '+%Y%m%d'` | ||
|
|
||
| # BUCKET_URL # from env (ex: "https://s3.ap-northeast-1.wasabisys.com") | ||
| # BUCKET_NAME # from env (ex: hoge-system-stg-bucket) | ||
| # BUCKET_DIR # from env (ex: fetcher/moneyforward) | ||
| # AWS_REGION # from env (ex: ap-northeast-1) | ||
| # AWS_ACCESS_KEY_ID # from env | ||
| # AWS_SECRET_ACCESS_KEY # from env | ||
| # user="xxxxxxxxx" # moneyforward id , from env | ||
| # pass="yyyyyyyyy" # moneyforward pass, from env | ||
| # wsAddr # from env (ex: localhost:7327) | ||
|
|
||
| SCRAPERS_BIN="/usr/local/bin/myscrapers" | ||
| AWS_BIN="/usr/local/bin/aws/dist/aws" | ||
| outputDir="/data/${YYYYMM}/${YYYYMMDD}" | ||
|
|
||
| REMOTE_DIR="${BUCKET_DIR}/${YYYYMM}/${YYYYMMDD}" | ||
|
|
||
| function download () { | ||
| echo "job start" | ||
| mkdir -p ${outputDir} | ||
| echo "output to dir: ${outputDir}" | ||
| outputDir=${outputDir} \ | ||
| user=${user} \ | ||
| pass=${pass} \ | ||
| ${SCRAPERS_BIN} download moneyforward --lastmonth | ||
| echo "job complete" | ||
| } | ||
|
|
||
| function create_s3_credentials () { | ||
| echo "s3 credentials create start" | ||
| mkdir -p ~/.aws/ | ||
|
|
||
| echo "[default]" >> ~/.aws/config | ||
| echo "region = ${AWS_REGION}" >> ~/.aws/config | ||
|
|
||
| echo "[default]" >> ~/.aws/credentials | ||
| echo "aws_access_key_id = ${AWS_ACCESS_KEY_ID}" >> ~/.aws/credentials | ||
| echo "aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY}" >> ~/.aws/credentials | ||
|
|
||
| chmod 400 ~/.aws/config | ||
| chmod 400 ~/.aws/credentials | ||
| ls -la ~/.aws/ | ||
| echo "s3 credentials create complete" | ||
| } | ||
|
|
||
| function s3_upload () { | ||
| echo "s3 upload start" | ||
| ${AWS_BIN} s3 cp ${outputDir}/ "s3://${BUCKET_NAME}/${REMOTE_DIR}" --recursive --endpoint-url="${BUCKET_URL}" | ||
| echo "s3 upload complete" | ||
| } | ||
|
|
||
| download | ||
|
|
||
| if [ -n $BUCKET_NAME ]; then | ||
| create_s3_credentials | ||
| s3_upload | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| ## moneyforward-cf | ||
|
|
||
| ### output CSV | ||
| 例： | ||
| ``` | ||
| 計算対象,日付,内容,金額（円）,保有金融機関,大項目,中項目,メモ,振替,削除 | ||
| ,07/16(火),ローソン,-291,三井住友カード,食費,食料品,,, | ||
| ,07/16(火),GITHUB,-158,JCBカード,通信費,情報サービス,,, | ||
| ,07/10(水),マクドナルド,-600,三井住友カード,食費,外食,,, | ||
| ``` | ||
|
|
||
| ### 出力先 | ||
| - コンテナ内デフォルト: `/data/YYYYMM/YYYYMMDD/cf.csv`, `--lastmonth` 付与時は、`/data/YYYYMM/YYYYMMDD/cf_lastmonth.csv` も出力。 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| package moneyforward | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "log/slog" | ||
| "myscrapers/internal/csv" | ||
| "strings" | ||
|
|
||
| "github.com/go-rod/rod" | ||
| ) | ||
|
|
||
| const cfFieldSize = 10 | ||
|
|
||
| func validateCF(header []string, bodies [][]string) error { | ||
| if len(header) != cfFieldSize { | ||
| return fmt.Errorf("invalid field size: header") | ||
| } | ||
| for i, b := range bodies { | ||
| if len(b) != cfFieldSize { | ||
| return fmt.Errorf("invalid field size: bodies #%d", i+1) | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // cfPage は /cf のページを rod で取得したもの | ||
| func getHeader(ctx context.Context, cfPage *rod.Page) (header []string, err error) { | ||
| cfDetailTable, err := cfPage.Element(`[id=cf-detail-table]`) | ||
| if err != nil { | ||
| slog.Error("failed to get cf-detail-table") | ||
| return []string{}, err | ||
| } | ||
|
|
||
| ths, err := cfDetailTable.Elements("th") | ||
| if err != nil { | ||
| slog.Error("failed to get cfDetailTable") | ||
| return []string{}, err | ||
| } | ||
|
|
||
| for _, th := range ths { | ||
| // セレクターの選択肢のテキストを消す | ||
| txt := strings.Split(th.MustText(), " ")[0] | ||
| // 改行を消す | ||
| txt = strings.ReplaceAll(txt, "\n", "") | ||
| // 無駄な空白を消す | ||
| txt = strings.ReplaceAll(txt, " ", "") | ||
| header = append(header, txt) | ||
| } | ||
|
|
||
| return header, nil | ||
| } | ||
|
|
||
| // cfPage は /cf のページを rod で取得したもの | ||
| func getBody(ctx context.Context, cfPage *rod.Page) (bodies [][]string, err error) { | ||
| cfDetailTable, err := cfPage.Element(`[id=cf-detail-table]`) | ||
| if err != nil { | ||
| slog.Error("failed to get cf-detail-table") | ||
| return [][]string{}, err | ||
| } | ||
|
|
||
| recordRows, err := cfDetailTable.Elements(`[class="transaction_list js-cf-edit-container target-active"`) // 1行ごとのrecordsのフィールドを特定する | ||
| if err != nil { | ||
| slog.Error("failed to get recordRows") | ||
| return [][]string{}, err | ||
| } | ||
|
|
||
| for _, recordRow := range recordRows { | ||
| var row []string | ||
| spans := recordRow.MustElements("td") // 1行ごとのレコードから各セルを抽出 | ||
| for _, span := range spans { | ||
| // セレクターはないので何も消さない | ||
| // 改行を消す | ||
| txt := strings.ReplaceAll(span.MustText(), "\n", "") | ||
| // 無駄な空白を消さない | ||
| row = append(row, txt) | ||
| } | ||
| bodies = append(bodies, row) | ||
| } | ||
|
|
||
| return bodies, nil | ||
| } | ||
|
|
||
| func ImportStart(ctx context.Context, filePath string, page *rod.Page) (err error) { | ||
| var header []string | ||
| var bodies [][]string | ||
|
|
||
| header, err = getHeader(ctx, page) | ||
| if err != nil { | ||
| slog.Error("failed to get header") | ||
| return err | ||
| } | ||
| slog.Info("get CSV header") | ||
|
|
||
| bodies, err = getBody(ctx, page) | ||
| if err != nil { | ||
| slog.Error("failed to get bodies") | ||
| return err | ||
| } | ||
| slog.Info("get CSV body") | ||
|
|
||
| // validation | ||
| if err := validateCF(header, bodies); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // csv書き込み | ||
| if err := csv.WriteFile(filePath, header, bodies); err != nil { | ||
| slog.Error("failed to output csv") | ||
| return err | ||
| } | ||
| slog.Info("output csv complete", "outputPath", filePath) | ||
| return nil | ||
| } |
Oops, something went wrong.