日拱一卒
0%

搭建 Smart Invest 架构

Posted on

ClickHouse

clickhouse/clickhouse-server

1
docker run -d -p 8123:8123 -p 9000:9000 -v /mnt/clickhouse-data:/var/lib/clickhouse/ -v /mnt/clickhouse-log:/var/log/clickhouse-server/ -e CLICKHOUSE_DB=smartinvest -e CLICKHOUSE_USER=smartinvest -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -e CLICKHOUSE_PASSWORD=smartinvest.123 --name smart-invest-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server:22.3-alpine

Python

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
sudo dnf groupinstall -y 'development tools'
sudo dnf install -y bzip2-devel expat-devel gdbm-devel ncurses-devel openssl-devel readline-devel sqlite-devel tk-devel xz-devel zlib-devel wget

cd /usr/local/src
wget https://www.python.org/ftp/python/3.8.13/Python-3.8.13.tgz
tar -zxvf Python-3.8.13.tgz

# ModuleNotFoundError: No module named 'xxx'
yum install -y openssl-devel bzip2-devel libffi-devel

# compile
cd Python-3.8.13/
./configure --prefix=/usr/local/python --with-ssl
./configure --enable-optimizations
make && make install

# upgrade pip
pip3 install --upgrade pip

ln -sf /usr/local/bin/python3 /usr/bin/python

AKShare

AKTools

Welcome to AKShare’s Online Documentation!

1
2
3
pip3 install --upgrade setuptools
pip3 install aktools -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host=mirrors.aliyun.com --upgrade
nohup python3 -m aktools > nohup.log 2>&1 &

Spring Cloud Data Flow

Batch-only Mode

10.2.2. MariaDB

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# download Spring Cloud Data Flow Server
wget https://repo.spring.io/release/org/springframework/cloud/spring-cloud-dataflow-server/2.9.2/spring-cloud-dataflow-server-2.9.2.jar

# setup mariadb
docker run -d --name task-db --env MARIADB_RANDOM_ROOT_PASSWORD="1" --env MARIADB_DATABASE=task --env MARIADB_USER=task --env MARIADB_PASSWORD=task.123 -v /mnt/task-db-data:/var/lib/mysql -p 3306:3306 mariadb:10.7

# setup jdk & maven
yum install -y java-17-openjdk maven
sudo alternatives --config java

# setup environment variables
export SPRING_CLOUD_DATAFLOW_FEATURES_STREAMS_ENABLED=false
export SPRING_CLOUD_DATAFLOW_FEATURES_SCHEDULES_ENABLED=false
export SPRING_CLOUD_DATAFLOW_FEATURES_TASKS_ENABLED=true

# running the server
# http://<ip>:9393/dashboard
nohup java -jar spring-cloud-dataflow-server-2.9.2.jar \
    --spring.datasource.url=jdbc:mariadb://localhost:3306/task?useMysqlMetadata=true \
    --spring.datasource.username=task \
    --spring.datasource.password=task.123 \
    --spring.datasource.driver-class-name=org.mariadb.jdbc.Driver \
    --spring.datasource.initialization.mode=always \
    --spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.MariaDB102Dialect \
    > ./nohup.log 2>&1 &

Register and Launch a Spring Cloud Task & Spring Batch

Batch Development

Multiple DataSources Sample Task

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
# build project
git clone https://github.com/csongyu/smart-invest.git

# install to local maven repository
mvn install:install-file -DgroupId=xyz.csongyu -DartifactId=smart-invest-task -Dversion=1.0.0 -Dpackaging=jar -DgeneratePom=true -Dfile=smart-invest-task-1.0.0.jar
mvn install:install-file -DgroupId=xyz.csongyu -DartifactId=smart-invest-job -Dversion=1.0.0 -Dpackaging=jar -DgeneratePom=true -Dfile=smart-invest-job-1.0.0.jar

# fund-name-em
# register applications
curl --location --request POST 'http://127.0.0.1:9393/apps/task/task-fund-name-em' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data-urlencode 'uri=maven://xyz.csongyu:smart-invest-task:1.0.0' \
--data-urlencode 'force=true'
curl --location --request POST 'http://127.0.0.1:9393/apps/task/task-initialize-schema-fund-name-em' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data-urlencode 'uri=maven://xyz.csongyu:smart-invest-task:1.0.0' \
--data-urlencode 'force=true'
curl --location --request POST 'http://127.0.0.1:9393/apps/task/job-fund-name-em' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data-urlencode 'uri=maven://xyz.csongyu:smart-invest-job:1.0.0' \
--data-urlencode 'force=true'
# create task
curl --location --request POST 'http://127.0.0.1:9393/tasks/definitions' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data-urlencode 'name=fne' \
--data-urlencode 'definition=task-fund-name-em && task-initialize-schema-fund-name-em && job-fund-name-em'

# fund-open-fund-info-em / unit-net-asset-value
# register applications
curl --location --request POST 'http://127.0.0.1:9393/apps/task/task-fund-open-fund-info-em-unit-net-asset-value' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data-urlencode 'uri=maven://xyz.csongyu:smart-invest-task:1.0.0' \
--data-urlencode 'force=true'
curl --location --request POST 'http://127.0.0.1:9393/apps/task/task-initialize-schema-fund-open-fund-info-em-unit-net-asset-value' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data-urlencode 'uri=maven://xyz.csongyu:smart-invest-task:1.0.0' \
--data-urlencode 'force=true'
curl --location --request POST 'http://127.0.0.1:9393/apps/task/job-fund-open-fund-info-em-unit-net-asset-value' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data-urlencode 'uri=maven://xyz.csongyu:smart-invest-job:1.0.0' \
--data-urlencode 'force=true'
# create task
curl --location --request POST 'http://127.0.0.1:9393/tasks/definitions' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data-urlencode 'name=fofiem' \
--data-urlencode 'definition=task-fund-open-fund-info-em-unit-net-asset-value && task-initialize-schema-fund-open-fund-info-em-unit-net-asset-value && job-fund-open-fund-info-em-unit-net-asset-value'

# fund-open-fund-daily-em
# register applications
curl --location --request POST 'http://127.0.0.1:9393/apps/task/task-fund-open-fund-daily-em' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data-urlencode 'uri=maven://xyz.csongyu:smart-invest-task:1.0.0' \
--data-urlencode 'force=true'
curl --location --request POST 'http://127.0.0.1:9393/apps/task/job-fund-open-fund-daily-em' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data-urlencode 'uri=maven://xyz.csongyu:smart-invest-job:1.0.0' \
--data-urlencode 'force=true'
# create task
curl --location --request POST 'http://127.0.0.1:9393/tasks/definitions' \
--header 'Content-Type: application/x-www-form-urlencoded' \
--data-urlencode 'name=fofde' \
--data-urlencode 'definition=task-fund-open-fund-daily-em && job-fund-open-fund-daily-em'

Scheduling

Batch Job Scheduling

Task Executions

Restrict Size of the Buffer Cache in Linux

1
2
3
4
5
6
7
8
9
10
# restrict buffer/cache
sync
echo 3 > /proc/sys/vm/drop_caches
echo 500 > /proc/sys/vm/vfs_cache_pressure

# build project
git clone https://github.com/csongyu/smart-invest.git

# launch application
nohup java -jar smart-invest-scheduler-1.0.0.jar > ./scheduler.log 2>&1 &