From 9115f502480e7bd1b539e67a7daf329f50e985ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Barkoczi?= Date: Tue, 21 Apr 2026 15:19:51 +0200 Subject: [PATCH] add gen --- .../LibriSpeech/create_bliss/train-clean-100 | 1 + .../LibriSpeech/create_bliss/train-clean-360 | 1 + .../LibriSpeech/create_bliss/train-other-500 | 1 + .../corpora/LibriSpeech/download/metadata_job | 1 + .../LibriSpeech/download/train-clean-100 | 1 + .../LibriSpeech/download/train-clean-360 | 1 + .../LibriSpeech/download/train-other-500 | 1 + .../ogg_conversion/train-clean-100 | 1 + .../ogg_conversion/train-clean-360 | 1 + .../ogg_conversion/train-other-500 | 1 + .../LibriSpeech/ogg_merge/train-other-960 | 1 + .../LibriSpeech/create_bliss/train-clean-100 | 1 + .../LibriSpeech/create_bliss/train-clean-360 | 1 + .../LibriSpeech/create_bliss/train-other-500 | 1 + .../LibriSpeech/download/metadata_job | 1 + .../LibriSpeech/download/train-clean-100 | 1 + .../LibriSpeech/download/train-clean-360 | 1 + .../LibriSpeech/download/train-other-500 | 1 + .../apply-g2p-for-train-clean-100 | 1 + .../apply-g2p-for-train-clean-360 | 1 + .../apply-g2p-for-train-other-500 | 1 + .../convert_bliss_lexicon_to_g2p_lexicon | 1 + .../extract-oov-from-train-clean-100 | 1 + .../extract-oov-from-train-clean-360 | 1 + .../extract-oov-from-train-other-500 | 1 + .../g2p-output-to-bliss-train-clean-100 | 1 + .../g2p-output-to-bliss-train-clean-360 | 1 + .../g2p-output-to-bliss-train-other-500 | 1 + .../folded_lexicon_with_unk/train_g2p_model | 1 + .../convert_text_to_bliss_lexicon_job | 1 + .../download_lexicon_job | 1 + .../merge_lexicon_job | 1 + .../remove_stress_marker | 1 + .../static_lexicon_job | 1 + .../ogg_conversion/train-clean-100 | 1 + .../ogg_conversion/train-clean-360 | 1 + .../ogg_conversion/train-other-500 | 1 + .../LibriSpeech/ogg_merge/train-other-960 | 1 + .../pca_fit_job | 1 + .../feature_dump_job | 1 + .../train-other-960/eow_returnn_vocab_job | 1 + .../pca_fit_job | 1 + .../feature_dump_job | 1 + .../train-other-960/eow_returnn_vocab_job | 1 + .../devtrain/forward_job | 1 + .../train-other-960/eow_returnn_vocab_job | 1 + .../train-other-960/train_devtrain_95_5_split | 1 + .../train/forward_job | 1 + .../ls960_wav2vec2_pca512/dump_features_job | 1 + .../ls960_wav2vec2_pca512/fit_pca/pca_fit_job | 1 + .../train-other-960/eow_returnn_vocab_job | 1 + .../train-other-960/train_devtrain_95_5_split | 1 + ...ain-other-960_train-other-960_filtered_eow | 1 + gen/recipe/i6_core | 1 + gen/recipe/i6_experiments | 1 + gen/settings.py | 301 ++++++++++++++++++ gen/sis | 1 + gen/sisyphus | 1 + gen/work | 1 + 59 files changed, 359 insertions(+) create mode 120000 gen/alias/corpora/LibriSpeech/create_bliss/train-clean-100 create mode 120000 gen/alias/corpora/LibriSpeech/create_bliss/train-clean-360 create mode 120000 gen/alias/corpora/LibriSpeech/create_bliss/train-other-500 create mode 120000 gen/alias/corpora/LibriSpeech/download/metadata_job create mode 120000 gen/alias/corpora/LibriSpeech/download/train-clean-100 create mode 120000 gen/alias/corpora/LibriSpeech/download/train-clean-360 create mode 120000 gen/alias/corpora/LibriSpeech/download/train-other-500 create mode 120000 gen/alias/corpora/LibriSpeech/ogg_conversion/train-clean-100 create mode 120000 gen/alias/corpora/LibriSpeech/ogg_conversion/train-clean-360 create mode 120000 gen/alias/corpora/LibriSpeech/ogg_conversion/train-other-500 create mode 120000 gen/alias/corpora/LibriSpeech/ogg_merge/train-other-960 create mode 120000 gen/alias/datasets/LibriSpeech/create_bliss/train-clean-100 create mode 120000 gen/alias/datasets/LibriSpeech/create_bliss/train-clean-360 create mode 120000 gen/alias/datasets/LibriSpeech/create_bliss/train-other-500 create mode 120000 gen/alias/datasets/LibriSpeech/download/metadata_job create mode 120000 gen/alias/datasets/LibriSpeech/download/train-clean-100 create mode 120000 gen/alias/datasets/LibriSpeech/download/train-clean-360 create mode 120000 gen/alias/datasets/LibriSpeech/download/train-other-500 create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/apply-g2p-for-train-clean-100 create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/apply-g2p-for-train-clean-360 create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/apply-g2p-for-train-other-500 create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/convert_bliss_lexicon_to_g2p_lexicon create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/extract-oov-from-train-clean-100 create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/extract-oov-from-train-clean-360 create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/extract-oov-from-train-other-500 create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/g2p-output-to-bliss-train-clean-100 create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/g2p-output-to-bliss-train-clean-360 create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/g2p-output-to-bliss-train-other-500 create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/train_g2p_model create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/convert_text_to_bliss_lexicon_job create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/download_lexicon_job create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/merge_lexicon_job create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/remove_stress_marker create mode 120000 gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/static_lexicon_job create mode 120000 gen/alias/datasets/LibriSpeech/ogg_conversion/train-clean-100 create mode 120000 gen/alias/datasets/LibriSpeech/ogg_conversion/train-clean-360 create mode 120000 gen/alias/datasets/LibriSpeech/ogg_conversion/train-other-500 create mode 120000 gen/alias/datasets/LibriSpeech/ogg_merge/train-other-960 create mode 120000 gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/debug_dev_clean_wav2vec2_hf_pca64_gaussians/hsmm.wav2vec2_hf_pca_dump.wav2vec2_large_pca64_stage1_fitpca_fullpass/pca_fit_job create mode 120000 gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/debug_dev_clean_wav2vec2_hf_pca64_gaussians/hsmm.wav2vec2_hf_pca_dump.wav2vec_large_pca64_stage2_feature_dump/feature_dump_job create mode 120000 gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/debug_dev_clean_wav2vec2_hf_pca64_gaussians/train-other-960/eow_returnn_vocab_job create mode 120000 gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/eow_phon_ls960_wav2vec2_hf_pca64_dump/hsmm.wav2vec2_hf_pca_dump.wav2vec2_large_pca64_stage1_fitpca_fullpass/pca_fit_job create mode 120000 gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/eow_phon_ls960_wav2vec2_hf_pca64_dump/hsmm.wav2vec2_hf_pca_dump.wav2vec_large_pca64_stage2_feature_dump/feature_dump_job create mode 120000 gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/eow_phon_ls960_wav2vec2_hf_pca64_dump/train-other-960/eow_returnn_vocab_job create mode 120000 gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/devtrain/forward_job create mode 120000 gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/train-other-960/eow_returnn_vocab_job create mode 120000 gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/train-other-960/train_devtrain_95_5_split create mode 120000 gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/train/forward_job create mode 120000 gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/dump_features_job create mode 120000 gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/fit_pca/pca_fit_job create mode 120000 gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/train-other-960/eow_returnn_vocab_job create mode 120000 gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/train-other-960/train_devtrain_95_5_split create mode 120000 gen/alias/train-other-960_train-other-960_filtered_eow create mode 160000 gen/recipe/i6_core create mode 160000 gen/recipe/i6_experiments create mode 100644 gen/settings.py create mode 120000 gen/sis create mode 160000 gen/sisyphus create mode 120000 gen/work diff --git a/gen/alias/corpora/LibriSpeech/create_bliss/train-clean-100 b/gen/alias/corpora/LibriSpeech/create_bliss/train-clean-100 new file mode 120000 index 0000000..7155b6f --- /dev/null +++ b/gen/alias/corpora/LibriSpeech/create_bliss/train-clean-100 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/datasets/librispeech/LibriSpeechCreateBlissCorpusJob.OjEfOC2QXh8l \ No newline at end of file diff --git a/gen/alias/corpora/LibriSpeech/create_bliss/train-clean-360 b/gen/alias/corpora/LibriSpeech/create_bliss/train-clean-360 new file mode 120000 index 0000000..465baa3 --- /dev/null +++ b/gen/alias/corpora/LibriSpeech/create_bliss/train-clean-360 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/datasets/librispeech/LibriSpeechCreateBlissCorpusJob.2GMuOxuirZVL \ No newline at end of file diff --git a/gen/alias/corpora/LibriSpeech/create_bliss/train-other-500 b/gen/alias/corpora/LibriSpeech/create_bliss/train-other-500 new file mode 120000 index 0000000..c06617b --- /dev/null +++ b/gen/alias/corpora/LibriSpeech/create_bliss/train-other-500 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/datasets/librispeech/LibriSpeechCreateBlissCorpusJob.N4devEBOAvgK \ No newline at end of file diff --git a/gen/alias/corpora/LibriSpeech/download/metadata_job b/gen/alias/corpora/LibriSpeech/download/metadata_job new file mode 120000 index 0000000..dbcc3ab --- /dev/null +++ b/gen/alias/corpora/LibriSpeech/download/metadata_job @@ -0,0 +1 @@ +/work/asr3/zyang/share/zhan/torch_setup/work/i6_core/datasets/librispeech/DownloadLibriSpeechMetadataJob.n7Yd9EbtVi13 \ No newline at end of file diff --git a/gen/alias/corpora/LibriSpeech/download/train-clean-100 b/gen/alias/corpora/LibriSpeech/download/train-clean-100 new file mode 120000 index 0000000..169977f --- /dev/null +++ b/gen/alias/corpora/LibriSpeech/download/train-clean-100 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/datasets/librispeech/DownloadLibriSpeechCorpusJob.4LL17D9Sz7NZ \ No newline at end of file diff --git a/gen/alias/corpora/LibriSpeech/download/train-clean-360 b/gen/alias/corpora/LibriSpeech/download/train-clean-360 new file mode 120000 index 0000000..d7eba02 --- /dev/null +++ b/gen/alias/corpora/LibriSpeech/download/train-clean-360 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/datasets/librispeech/DownloadLibriSpeechCorpusJob.QLzK7S51OV6m \ No newline at end of file diff --git a/gen/alias/corpora/LibriSpeech/download/train-other-500 b/gen/alias/corpora/LibriSpeech/download/train-other-500 new file mode 120000 index 0000000..f480017 --- /dev/null +++ b/gen/alias/corpora/LibriSpeech/download/train-other-500 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/datasets/librispeech/DownloadLibriSpeechCorpusJob.8iqSB1tz3OMD \ No newline at end of file diff --git a/gen/alias/corpora/LibriSpeech/ogg_conversion/train-clean-100 b/gen/alias/corpora/LibriSpeech/ogg_conversion/train-clean-100 new file mode 120000 index 0000000..2cfa7d7 --- /dev/null +++ b/gen/alias/corpora/LibriSpeech/ogg_conversion/train-clean-100 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/audio/encoding/BlissChangeEncodingJob.R3T4Yu1U9wHG \ No newline at end of file diff --git a/gen/alias/corpora/LibriSpeech/ogg_conversion/train-clean-360 b/gen/alias/corpora/LibriSpeech/ogg_conversion/train-clean-360 new file mode 120000 index 0000000..5441bfa --- /dev/null +++ b/gen/alias/corpora/LibriSpeech/ogg_conversion/train-clean-360 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/audio/encoding/BlissChangeEncodingJob.NIf5NG6AJsbD \ No newline at end of file diff --git a/gen/alias/corpora/LibriSpeech/ogg_conversion/train-other-500 b/gen/alias/corpora/LibriSpeech/ogg_conversion/train-other-500 new file mode 120000 index 0000000..063560a --- /dev/null +++ b/gen/alias/corpora/LibriSpeech/ogg_conversion/train-other-500 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/audio/encoding/BlissChangeEncodingJob.gjPZrjES7C9A \ No newline at end of file diff --git a/gen/alias/corpora/LibriSpeech/ogg_merge/train-other-960 b/gen/alias/corpora/LibriSpeech/ogg_merge/train-other-960 new file mode 120000 index 0000000..5db1c82 --- /dev/null +++ b/gen/alias/corpora/LibriSpeech/ogg_merge/train-other-960 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/corpus/transform/MergeCorporaJob.hlZ8ixhLSaaQ \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/create_bliss/train-clean-100 b/gen/alias/datasets/LibriSpeech/create_bliss/train-clean-100 new file mode 120000 index 0000000..7155b6f --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/create_bliss/train-clean-100 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/datasets/librispeech/LibriSpeechCreateBlissCorpusJob.OjEfOC2QXh8l \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/create_bliss/train-clean-360 b/gen/alias/datasets/LibriSpeech/create_bliss/train-clean-360 new file mode 120000 index 0000000..465baa3 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/create_bliss/train-clean-360 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/datasets/librispeech/LibriSpeechCreateBlissCorpusJob.2GMuOxuirZVL \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/create_bliss/train-other-500 b/gen/alias/datasets/LibriSpeech/create_bliss/train-other-500 new file mode 120000 index 0000000..c06617b --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/create_bliss/train-other-500 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/datasets/librispeech/LibriSpeechCreateBlissCorpusJob.N4devEBOAvgK \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/download/metadata_job b/gen/alias/datasets/LibriSpeech/download/metadata_job new file mode 120000 index 0000000..dbcc3ab --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/download/metadata_job @@ -0,0 +1 @@ +/work/asr3/zyang/share/zhan/torch_setup/work/i6_core/datasets/librispeech/DownloadLibriSpeechMetadataJob.n7Yd9EbtVi13 \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/download/train-clean-100 b/gen/alias/datasets/LibriSpeech/download/train-clean-100 new file mode 120000 index 0000000..169977f --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/download/train-clean-100 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/datasets/librispeech/DownloadLibriSpeechCorpusJob.4LL17D9Sz7NZ \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/download/train-clean-360 b/gen/alias/datasets/LibriSpeech/download/train-clean-360 new file mode 120000 index 0000000..d7eba02 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/download/train-clean-360 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/datasets/librispeech/DownloadLibriSpeechCorpusJob.QLzK7S51OV6m \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/download/train-other-500 b/gen/alias/datasets/LibriSpeech/download/train-other-500 new file mode 120000 index 0000000..f480017 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/download/train-other-500 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/datasets/librispeech/DownloadLibriSpeechCorpusJob.8iqSB1tz3OMD \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/apply-g2p-for-train-clean-100 b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/apply-g2p-for-train-clean-100 new file mode 120000 index 0000000..d421354 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/apply-g2p-for-train-clean-100 @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/g2p/apply/ApplyG2PModelJob.UkZC6aXDsSpJ \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/apply-g2p-for-train-clean-360 b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/apply-g2p-for-train-clean-360 new file mode 120000 index 0000000..d04d277 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/apply-g2p-for-train-clean-360 @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/g2p/apply/ApplyG2PModelJob.0wLJiym2NHHq \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/apply-g2p-for-train-other-500 b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/apply-g2p-for-train-other-500 new file mode 120000 index 0000000..db2e01f --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/apply-g2p-for-train-other-500 @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/g2p/apply/ApplyG2PModelJob.zjzPW918CJDL \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/convert_bliss_lexicon_to_g2p_lexicon b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/convert_bliss_lexicon_to_g2p_lexicon new file mode 120000 index 0000000..1cfbf83 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/convert_bliss_lexicon_to_g2p_lexicon @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/g2p/convert/BlissLexiconToG2PLexiconJob.kqQmbqufPzZ3 \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/extract-oov-from-train-clean-100 b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/extract-oov-from-train-clean-100 new file mode 120000 index 0000000..c93983f --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/extract-oov-from-train-clean-100 @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/corpus/stats/ExtractOovWordsFromCorpusJob.PFT8D7dpU42L \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/extract-oov-from-train-clean-360 b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/extract-oov-from-train-clean-360 new file mode 120000 index 0000000..031f2cc --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/extract-oov-from-train-clean-360 @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/corpus/stats/ExtractOovWordsFromCorpusJob.2tTlOaBZmyPe \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/extract-oov-from-train-other-500 b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/extract-oov-from-train-other-500 new file mode 120000 index 0000000..5bd70f9 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/extract-oov-from-train-other-500 @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/corpus/stats/ExtractOovWordsFromCorpusJob.wKXQwE3z0oxe \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/g2p-output-to-bliss-train-clean-100 b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/g2p-output-to-bliss-train-clean-100 new file mode 120000 index 0000000..9e641f0 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/g2p-output-to-bliss-train-clean-100 @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/g2p/convert/G2POutputToBlissLexiconJob.l47vFd9ytgzv \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/g2p-output-to-bliss-train-clean-360 b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/g2p-output-to-bliss-train-clean-360 new file mode 120000 index 0000000..9cc96d1 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/g2p-output-to-bliss-train-clean-360 @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/g2p/convert/G2POutputToBlissLexiconJob.wYfSVnneAI4B \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/g2p-output-to-bliss-train-other-500 b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/g2p-output-to-bliss-train-other-500 new file mode 120000 index 0000000..179aaf5 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/g2p-output-to-bliss-train-other-500 @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/g2p/convert/G2POutputToBlissLexiconJob.ac0qjxfECRbz \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/train_g2p_model b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/train_g2p_model new file mode 120000 index 0000000..0959f59 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unk/train_g2p_model @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/g2p/train/TrainG2PModelJob.pD4nbqFLWtbi \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/convert_text_to_bliss_lexicon_job b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/convert_text_to_bliss_lexicon_job new file mode 120000 index 0000000..322517c --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/convert_text_to_bliss_lexicon_job @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/lexicon/conversion/LexiconFromTextFileJob.mTRl42KFeZSx \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/download_lexicon_job b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/download_lexicon_job new file mode 120000 index 0000000..2149ce9 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/download_lexicon_job @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/tools/download/DownloadJob.0UXAqd5DuQG7 \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/merge_lexicon_job b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/merge_lexicon_job new file mode 120000 index 0000000..4d9e76d --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/merge_lexicon_job @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/lexicon/modification/MergeLexiconJob.qKaOAPqURCkK \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/remove_stress_marker b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/remove_stress_marker new file mode 120000 index 0000000..ef45815 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/remove_stress_marker @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/text/processing/PipelineJob.p4BOP5qZ6T1G \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/static_lexicon_job b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/static_lexicon_job new file mode 120000 index 0000000..80766cf --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/folded_lexicon_with_unknown/static_lexicon_job @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/lexicon/modification/WriteLexiconJob.3Ih8wASQiD3q \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/ogg_conversion/train-clean-100 b/gen/alias/datasets/LibriSpeech/ogg_conversion/train-clean-100 new file mode 120000 index 0000000..2cfa7d7 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/ogg_conversion/train-clean-100 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/audio/encoding/BlissChangeEncodingJob.R3T4Yu1U9wHG \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/ogg_conversion/train-clean-360 b/gen/alias/datasets/LibriSpeech/ogg_conversion/train-clean-360 new file mode 120000 index 0000000..5441bfa --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/ogg_conversion/train-clean-360 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/audio/encoding/BlissChangeEncodingJob.NIf5NG6AJsbD \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/ogg_conversion/train-other-500 b/gen/alias/datasets/LibriSpeech/ogg_conversion/train-other-500 new file mode 120000 index 0000000..063560a --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/ogg_conversion/train-other-500 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/audio/encoding/BlissChangeEncodingJob.gjPZrjES7C9A \ No newline at end of file diff --git a/gen/alias/datasets/LibriSpeech/ogg_merge/train-other-960 b/gen/alias/datasets/LibriSpeech/ogg_merge/train-other-960 new file mode 120000 index 0000000..5db1c82 --- /dev/null +++ b/gen/alias/datasets/LibriSpeech/ogg_merge/train-other-960 @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/corpus/transform/MergeCorporaJob.hlZ8ixhLSaaQ \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/debug_dev_clean_wav2vec2_hf_pca64_gaussians/hsmm.wav2vec2_hf_pca_dump.wav2vec2_large_pca64_stage1_fitpca_fullpass/pca_fit_job b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/debug_dev_clean_wav2vec2_hf_pca64_gaussians/hsmm.wav2vec2_hf_pca_dump.wav2vec2_large_pca64_stage1_fitpca_fullpass/pca_fit_job new file mode 120000 index 0000000..e49c97d --- /dev/null +++ b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/debug_dev_clean_wav2vec2_hf_pca64_gaussians/hsmm.wav2vec2_hf_pca_dump.wav2vec2_large_pca64_stage1_fitpca_fullpass/pca_fit_job @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/returnn/forward/ReturnnForwardJobV2.tnW0NGPZhTkp \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/debug_dev_clean_wav2vec2_hf_pca64_gaussians/hsmm.wav2vec2_hf_pca_dump.wav2vec_large_pca64_stage2_feature_dump/feature_dump_job b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/debug_dev_clean_wav2vec2_hf_pca64_gaussians/hsmm.wav2vec2_hf_pca_dump.wav2vec_large_pca64_stage2_feature_dump/feature_dump_job new file mode 120000 index 0000000..6aad1bf --- /dev/null +++ b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/debug_dev_clean_wav2vec2_hf_pca64_gaussians/hsmm.wav2vec2_hf_pca_dump.wav2vec_large_pca64_stage2_feature_dump/feature_dump_job @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/returnn/forward/ReturnnForwardJobV2.xWpTuhforArg \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/debug_dev_clean_wav2vec2_hf_pca64_gaussians/train-other-960/eow_returnn_vocab_job b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/debug_dev_clean_wav2vec2_hf_pca64_gaussians/train-other-960/eow_returnn_vocab_job new file mode 120000 index 0000000..36fcfb3 --- /dev/null +++ b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/debug_dev_clean_wav2vec2_hf_pca64_gaussians/train-other-960/eow_returnn_vocab_job @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/returnn/vocabulary/ReturnnVocabFromPhonemeInventory.opHncyhHoqsj \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/eow_phon_ls960_wav2vec2_hf_pca64_dump/hsmm.wav2vec2_hf_pca_dump.wav2vec2_large_pca64_stage1_fitpca_fullpass/pca_fit_job b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/eow_phon_ls960_wav2vec2_hf_pca64_dump/hsmm.wav2vec2_hf_pca_dump.wav2vec2_large_pca64_stage1_fitpca_fullpass/pca_fit_job new file mode 120000 index 0000000..d90108b --- /dev/null +++ b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/eow_phon_ls960_wav2vec2_hf_pca64_dump/hsmm.wav2vec2_hf_pca_dump.wav2vec2_large_pca64_stage1_fitpca_fullpass/pca_fit_job @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/returnn/forward/ReturnnForwardJobV2.bVZ3dWfWdnSx \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/eow_phon_ls960_wav2vec2_hf_pca64_dump/hsmm.wav2vec2_hf_pca_dump.wav2vec_large_pca64_stage2_feature_dump/feature_dump_job b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/eow_phon_ls960_wav2vec2_hf_pca64_dump/hsmm.wav2vec2_hf_pca_dump.wav2vec_large_pca64_stage2_feature_dump/feature_dump_job new file mode 120000 index 0000000..f6fbb9b --- /dev/null +++ b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/eow_phon_ls960_wav2vec2_hf_pca64_dump/hsmm.wav2vec2_hf_pca_dump.wav2vec_large_pca64_stage2_feature_dump/feature_dump_job @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/returnn/forward/ReturnnForwardJobV2.ZJ2SZ5NqIaTN \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/eow_phon_ls960_wav2vec2_hf_pca64_dump/train-other-960/eow_returnn_vocab_job b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/eow_phon_ls960_wav2vec2_hf_pca64_dump/train-other-960/eow_returnn_vocab_job new file mode 120000 index 0000000..36fcfb3 --- /dev/null +++ b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/eow_phon_ls960_wav2vec2_hf_pca64_dump/train-other-960/eow_returnn_vocab_job @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/returnn/vocabulary/ReturnnVocabFromPhonemeInventory.opHncyhHoqsj \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/devtrain/forward_job b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/devtrain/forward_job new file mode 120000 index 0000000..19673f8 --- /dev/null +++ b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/devtrain/forward_job @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/returnn/forward/ReturnnForwardJobV2.FoYysJWDOQoJ \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/train-other-960/eow_returnn_vocab_job b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/train-other-960/eow_returnn_vocab_job new file mode 120000 index 0000000..36fcfb3 --- /dev/null +++ b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/train-other-960/eow_returnn_vocab_job @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/returnn/vocabulary/ReturnnVocabFromPhonemeInventory.opHncyhHoqsj \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/train-other-960/train_devtrain_95_5_split b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/train-other-960/train_devtrain_95_5_split new file mode 120000 index 0000000..1015023 --- /dev/null +++ b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/train-other-960/train_devtrain_95_5_split @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/corpus/segments/ShuffleAndSplitSegmentsJob.VpSki93GDEUE \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/train/forward_job b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/train/forward_job new file mode 120000 index 0000000..855ff35 --- /dev/null +++ b/gen/alias/example_setups/librispeech/ctc_rnnt_standalone_2024/ls960_ctc_eow_phon_training_dataset_95_5_split_with_optional_hdf_debug/train/forward_job @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/returnn/forward/ReturnnForwardJobV2.FbDteqkrTBdU \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/dump_features_job b/gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/dump_features_job new file mode 120000 index 0000000..bf51532 --- /dev/null +++ b/gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/dump_features_job @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/returnn/forward/ReturnnForwardJobV2.BOMCNfbdgces \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/fit_pca/pca_fit_job b/gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/fit_pca/pca_fit_job new file mode 120000 index 0000000..f720267 --- /dev/null +++ b/gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/fit_pca/pca_fit_job @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/returnn/forward/ReturnnForwardJobV2.x9thBehqm2JN \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/train-other-960/eow_returnn_vocab_job b/gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/train-other-960/eow_returnn_vocab_job new file mode 120000 index 0000000..36fcfb3 --- /dev/null +++ b/gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/train-other-960/eow_returnn_vocab_job @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/returnn/vocabulary/ReturnnVocabFromPhonemeInventory.opHncyhHoqsj \ No newline at end of file diff --git a/gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/train-other-960/train_devtrain_95_5_split b/gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/train-other-960/train_devtrain_95_5_split new file mode 120000 index 0000000..1015023 --- /dev/null +++ b/gen/alias/example_setups/librispeech/feature_dump/ls960_wav2vec2_pca512/train-other-960/train_devtrain_95_5_split @@ -0,0 +1 @@ +/work/asr3/zyang/share/joerg/sisyphus_work_dirs/gen/i6_core/corpus/segments/ShuffleAndSplitSegmentsJob.VpSki93GDEUE \ No newline at end of file diff --git a/gen/alias/train-other-960_train-other-960_filtered_eow b/gen/alias/train-other-960_train-other-960_filtered_eow new file mode 120000 index 0000000..9292654 --- /dev/null +++ b/gen/alias/train-other-960_train-other-960_filtered_eow @@ -0,0 +1 @@ +/work/asr4/zyang/mini/work/i6_core/returnn/oggzip/BlissToOggZipJob.sxQSaoYtXDSp \ No newline at end of file diff --git a/gen/recipe/i6_core b/gen/recipe/i6_core new file mode 160000 index 0000000..c7c3551 --- /dev/null +++ b/gen/recipe/i6_core @@ -0,0 +1 @@ +Subproject commit c7c355176922c182e57ab2dd2e9c838174f0a5a1 diff --git a/gen/recipe/i6_experiments b/gen/recipe/i6_experiments new file mode 160000 index 0000000..25efbd2 --- /dev/null +++ b/gen/recipe/i6_experiments @@ -0,0 +1 @@ +Subproject commit 25efbd2c85d41f8631486099e968470723669229 diff --git a/gen/settings.py b/gen/settings.py new file mode 100644 index 0000000..65d11f5 --- /dev/null +++ b/gen/settings.py @@ -0,0 +1,301 @@ +# Sisyphus settings file +import getpass +import os.path +import socket + +# import sys + +# sys.path.append("/u/beck/dev/cachemanager/") +############################# +# Local Settings File Options +############################# + +# can be "apptainer", "docker", "singularity" or None +CONTAINER_MODE = "apptainer" + +# local path or e.g. docker registry image path +# CONTAINER_IMAGE = "/work/asr4/rossenbach/rescale/pytorch_mixed_precision/apptainer/u22_pytorch2.1_onnx_flashlight_0224_jaist_project.sif" +# diff +# CONTAINER_IMAGE = "/work/asr4/rossenbach/rescale/pytorch_mixed_precision/apptainer/u22cuda12.1_pytorch2.2_onnxrt_1.17.1_flashlight_warprnnt_0624.sif" + +# runable one - 128baseline + lm +#CONTAINER_IMAGE = "/work/asr4/rossenbach/rescale/pytorch_mixed_precision/apptainer/u22cuda12.1_pytorch2.2_onnxrt_1.17.1_flashlight_warprnnt_0624v3.sif" +# torch 2.7 for wav2vec2 +CONTAINER_IMAGE = "/work/asr4/zyang/images/u22cuda12.1_pytorch2.7_onnxrt_1.17.1_flashlight_warprnnt_0624v3_huggingface.sif" +# simon +# CONTAINER_IMAGE = "/work/asr4/berger/apptainer/images/torch-2.8_onnx-1.22.sif" + +# file systems to bind in a ":" format +CONTAINER_BINDS = ["/work/asr4", "/work/asr3", "/work/common", "/work/tools22", "/u/corpora", "/work/smt4", + "/work/tools", "/u/jxu", "/u/rossenbach", "/u/joerg.barkoczi", "/u/berger", "/u/zyang", "/u/zeyer", "/u/zeineldeen", "/u/zhan.shu","/run", "/u/enrique.leon.lozano"] + +# can be "sge", "slurm" or "pbs" (pbs is experimental) +SUBMIT_ENGINE = "slurm" + +# hostname or ip of machine to use for cluster access +SUBMIT_GATEWAY = "cn-04" + +# the username +USER = "joerg.barkoczi" + +# List if extra env vars to set "before" sisyphus execution +# Can for example be used to set the PYTHONPATH to a custom sisyphus +# different to the one installed in the container for debugging / development purposes +EXTRA_ENV = [ + "PYTHONNOUSERSITE=1", +] + +######################### +# Setup Specific Settings +######################### + +# Unfortunately needed when using g2p from image: +# diff: but never used +G2P_PATH = "/usr/local/bin/g2p.py" + +########################## +# Sisyphus Global Settings +########################## +try: + MAIL_ADDRESS = getpass.getuser() +except KeyError: + MAIL_ADDRESS = None + +JOB_USE_TAGS_IN_PATH = False +JOB_AUTO_CLEANUP = False +SHOW_JOB_TARGETS = False +PRINT_ERROR = False +DELAYED_CHECK_FOR_WORKER = False + +WARNING_ABSPATH = False + +SHORT_JOB_NAMES = True + +# For debugging to 1 +# GRAPH_WORKER = 1 + +DEFAULT_ENVIRONMENT_KEEP = {'CUDA_VISIBLE_DEVICES', 'HOME', 'PWD', 'SGE_STDERR_PATH', 'SGE_TASK_ID', 'TMP', 'TMPDIR', + 'USER', "LD_LIBRARY_PATH"} + +DEFAULT_ENVIRONMENT_SET = { + 'LANG': 'en_US.UTF-8', + 'MKL_NUM_THREADS': 2, + 'OMP_NUM_THREADS': 2, + 'PATH': ':'.join(['/usr/local/sbin', '/usr/local/bin', + '/usr/sbin', '/usr/bin', + '/sbin', '/bin']), + 'SHELL': '/bin/bash', + "NUMBA_CACHE_DIR": f"/var/tmp/numba_cache_{USER}", # used for librosa + "PYTORCH_KERNEL_CACHE_PATH": f"/var/tmp/", # used for cuda pytorch + # diff + "MQTT_USERNAME": "i6", + "MQTT_PASSWORD": "1801", + "PYTHONNOUSERSITE": "1", # used for huggingface image +} + + +########################### +# Sisyphus Code Definitions +########################### + +def engine(): + from sisyphus.engine import EngineSelector + from sisyphus.localengine import LocalEngine + from sisyphus.simple_linux_utility_for_resource_management_engine import SimpleLinuxUtilityForResourceManagementEngine + + temp_exclude = [ + 257, + 231, # works but very slow? + 240, # random mem illegal access + 222, + 282, + 283, + 803, # doesnt produce output + 601, # doesnt produce output + 602, # doesnt produce output + ] + + default_rqmt={'cpu' : 1, 'mem' : 2, 'gpu' : 0, 'time' : 1} + + if temp_exclude: + default_rqmt["sbatch_args"] = ["-x", ",".join([f"cn-{node}" for node in temp_exclude])] + + # diff: only slurm + return EngineSelector( + engines={ + 'short': SimpleLinuxUtilityForResourceManagementEngine( + default_rqmt=default_rqmt, + gateway=SUBMIT_GATEWAY, + ), + 'long': SimpleLinuxUtilityForResourceManagementEngine( + default_rqmt=default_rqmt, + gateway=SUBMIT_GATEWAY, + )}, + default_engine='long' + ) + +# def engine(): +# from sisyphus.engine import EngineSelector +# from sisyphus.localengine import LocalEngine +# from sisyphus.simple_linux_utility_for_resource_management_engine import ( +# SimpleLinuxUtilityForResourceManagementEngine, +# ) + +# temp_exclude = [ +# 257, +# 231, # works but very slow? +# 240, # random mem illegal access +# 222, +# 283, +# 803, # doesnt produce output +# 601, # doesnt produce output +# 602, # doesnt produce output +# ] + +# default_rqmt = {"cpu": 1, "mem": 4, "time": 1} +# if temp_exclude: +# default_rqmt["sbatch_args"] = ["-x", ",".join([f"cn-{node}" for node in temp_exclude])] + +# return EngineSelector( +# engines={ +# "short": LocalEngine(cpus=4), +# "long": SimpleLinuxUtilityForResourceManagementEngine(default_rqmt=default_rqmt), +# }, +# default_engine="long", +# ) + +# def check_engine_limits(current_rqmt, task): +# """ +# i6 support for gpu_mem +# """ +# current_rqmt['time'] = min(168, current_rqmt.get('time', 2)) +# if current_rqmt.get('gpu', 0) > 0 and '-p' not in current_rqmt.get('sbatch_args', []): +# if current_rqmt.get("gpu_mem", 0) > 24: +# current_rqmt['sbatch_args'] = ['-p', 'gpu_48gb'] +# elif current_rqmt.get("gpu_mem", 0) > 11: +# current_rqmt['sbatch_args'] = ['-p', 'gpu_24gb', "--exclude=cn-233"] +# else: +# current_rqmt['sbatch_args'] = ['-p', 'gpu_11gb'] + +# ############################################# +# # Example to set settings based on alias name +# ############################################# + +# #aliases = [] +# #for prefix in list(task._job._sis_alias_prefixes) + [""]: +# # for alias in task._job.get_aliases() or [""]: +# # aliases.append(prefix + alias) + +# #if "gmm_align" in "\t".join(aliases or ""): +# # current_rqmt['sbatch_args'] = ['-p', 'cpu_slow'] + +# return current_rqmt + +CPU_SHORT_JOBLIST = ["AverageTFCheckpointsJob", "GetBestTFCheckpointJob"] +def check_engine_limits(current_rqmt, task): + current_rqmt["time"] = min(168, current_rqmt.get("time", 2)) + curr_sbatch_args = current_rqmt.get("sbatch_args", []) + if "-p" not in current_rqmt.get("sbatch_args", []): + if current_rqmt.get("gpu", 0) > 0: + # gpu + if current_rqmt["time"] <= 1 and current_rqmt.get("gpu_mem", 0) <= 24: + current_rqmt["sbatch_args"] = ["-p", "gpu_test_24gb"] + curr_sbatch_args + elif current_rqmt.get("gpu_mem", 0) > 24: + current_rqmt["sbatch_args"] = ["-p", "gpu_48gb"] + curr_sbatch_args # ["-p", "gpu_80gb"] + curr_sbatch_args + elif current_rqmt.get("gpu_mem", 0) > 11: + current_rqmt["sbatch_args"] = ["-p", "gpu_24gb"] + curr_sbatch_args + else: + current_rqmt["sbatch_args"] = ["-p", "gpu_11gb"] + curr_sbatch_args + else: + # cpu with SSE4 and AVX + if task._job.__class__.__name__ in CPU_SHORT_JOBLIST: + current_rqmt["sbatch_args"] = ["-p", "cpu_short"] + curr_sbatch_args + return current_rqmt + + +def build_apptainer_command(call): + """ + Apptainer specific launch code + """ + command = [] + if EXTRA_ENV: + command += ["env"] + EXTRA_ENV + command += ["apptainer", "exec", "--nv"] + for bind in CONTAINER_BINDS: + command += ["--bind", bind] + command += [CONTAINER_IMAGE] + return command + ["sis"] + call[2:] + + +def build_singularity_command(call): + """ + Singularity specific launch code + """ + command = [] + if EXTRA_ENV: + command += ["env"] + EXTRA_ENV + command += ["singularity", "exec", "--nv"] + for bind in CONTAINER_BINDS: + command += ["--bind", bind] + command += [CONTAINER_IMAGE] + return command + ["sis"] + call[2:] + + +def build_docker_command(call): + """ + Docker specific launch code + """ + from pwd import getpwnam + # diff: USER, 's2322008' + userid, groupid = getpwnam('s2322008')[2:4] + exp_dir = os.path.dirname(__file__) + command = [ + "docker", "run", "-t", + "--rm", # delete container after execution + "-u", "%i:%i" % (userid, groupid), # passing the username directly does not work with LDAP users + "--runtime=nvidia", + "--gpus", "device=%i" % gpu, + "-m", "32g", + "--shm-size", "16g", + "-w", exp_dir, + ] + for env in EXTRA_ENV: + command += ["-e", env] + for bind in CONTAINER_BINDS: + command += ["-v", bind] + command += [CONTAINER_IMAGE] + command += ["sh", "-e", "-c"] + return command + ["sis"] + call[2:] + + +def worker_wrapper(job, task_name, call): + """ + All worker calls are passed through this function. + Is used to wrap the execution call with the correct container command. + Usually it is not necessary to alter things here, + but any worker call can be fully customized here. + """ + from sisyphus.engine import EngineSelector + from sisyphus.localengine import LocalEngine + app_blacklist = ["FairseqHydraTrainWav2VecUJob"] + if type(job).__name__ in app_blacklist: + return call + + if CONTAINER_MODE == "apptainer": + command = build_apptainer_command(call) + elif CONTAINER_MODE == "docker": + command = build_docker_command(call) + elif CONTAINER_MODE == "singularity": + command = build_singularity_command(call) + else: + raise ValueError("Invalid CONTAINER_MODE %s" % CONTAINER_MODE) + + ts = {t.name(): t for t in job.tasks()} + t = ts[task_name] + e = engine() # Usually EngineSelector, but can be LocalEngine if no settings file is present + if isinstance(e, EngineSelector): + e = engine().get_used_engine_by_rqmt(t.rqmt()) + if isinstance(e, LocalEngine): + return call + else: + return command diff --git a/gen/sis b/gen/sis new file mode 120000 index 0000000..4ed1ebc --- /dev/null +++ b/gen/sis @@ -0,0 +1 @@ +sisyphus/sis \ No newline at end of file diff --git a/gen/sisyphus b/gen/sisyphus new file mode 160000 index 0000000..b380df2 --- /dev/null +++ b/gen/sisyphus @@ -0,0 +1 @@ +Subproject commit b380df264ccbd53c46c3cf1e4d77726c559117a2 diff --git a/gen/work b/gen/work new file mode 120000 index 0000000..256f6be --- /dev/null +++ b/gen/work @@ -0,0 +1 @@ +/u/joerg.barkoczi/work/sisyphus_work_dirs/gen \ No newline at end of file