-
Notifications
You must be signed in to change notification settings - Fork 3
/
word2vec_trainer.yaml
67 lines (65 loc) · 2.13 KB
/
word2vec_trainer.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
apiVersion: batch/v1
kind: Job
metadata:
name: fasttext-trainer
spec:
template:
spec:
volumes:
- name: shared-volume
emptyDir: {}
- name: gcloud-storage-secret
secret:
secretName: backup-manager-secret
- name: script
configMap:
name: train-word2vec
defaultMode: 0744
- name: local-settings-noah
secret:
secretName: local-settings-noah
initContainers:
- name: downloader
image: google/cloud-sdk
volumeMounts:
- name: shared-volume
mountPath: /storage
- name: gcloud-storage-secret
mountPath: /conf
env:
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /conf/BackupManagerKey.json
command: ["bash"]
args: ["-c", "gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS} && gsutil cp gs://development-research/sefaria-export_prefix_refs.json /storage/sefaria-export_prefix_refs.json"]
- name: training
image: gcr.io/production-deployment/multi_monitor:v2.4.0
volumeMounts:
- name: shared-volume
mountPath: /storage
- name: script
mountPath: /scripts
- mountPath: /settings
name: local-settings-noah
command: ["bash"]
args: ["-c", "pip install gensim && python /scripts/Word2Vec.py -f /storage/sefaria-export_prefix_refs.json"]
containers:
- name: uploader
resources:
limits:
memory: "20.0Gi"
cpu: "3500m"
requests:
memory: "20.0Gi"
cpu: "3500m"
image: google/cloud-sdk
volumeMounts:
- name: shared-volume
mountPath: /storage
- name: gcloud-storage-secret
mountPath: /conf
env:
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /conf/BackupManagerKey.json
command: ["bash"]
args: ["-c", "gcloud auth activate-service-account --key-file ${GOOGLE_APPLICATION_CREDENTIALS} && gsutil cp OUTPUT_LOCATION_WORD2VEC gs://development-research/word2vec.out"]
restartPolicy: Never