feat: add mfai package and dependencies (backport #251) (#252)

<hr>This is an automatic backport of pull request #251 done by [Mergify](https://mergify.com).

Author: mergify[bot]

.metwork-framework/components.md

@@ -1,7 +1,11 @@
 | Name | Version | Layer |
 | --- | --- | --- |
 | [absl-py](https://github.com/abseil/abseil-py) | 2.2.2 | python3_ia |
+| [antlr4-python3-runtime](http://www.antlr.org) | 4.9.3 | python3_ia |
 | [astunparse](https://github.com/simonpercivall/astunparse) | 1.6.3 | python3_ia |
+| [axial_attention](https://github.com/lucidrains/axial-attention) | 0.6.1 | python3_ia |
+| [bitsandbytes](https://github.com/bitsandbytes-foundation/bitsandbytes) | 0.49.1 | python3_ia |
+| [dataclasses-json](https://github.com/lidatong/dataclasses-json) | 0.6.7 | python3_ia |
 | [diffusers](https://github.com/huggingface/diffusers) | 0.33.1 | python3_ia |
 | [dm-tree](https://github.com/deepmind/tree) | 0.1.9 | python3_ia |
 | [einops](https://github.com/arogozhnikov/einops) | 0.8.1 | python3_ia |
@@ -11,12 +15,19 @@
 | [google-pasta](https://github.com/google/pasta) | 0.2.0 | python3_ia |
 | [grpcio](https://grpc.io) | 1.71.0 | python3_ia |
 | [huggingface_hub](https://github.com/huggingface/huggingface_hub) | 0.31.4 | python3_ia |
+| [hydra-core](https://github.com/facebookresearch/hydra) | 1.3.2 | python3_ia |
+| [importlib_resources](https://pypi.org/project/importlib_resources) | 6.5.2 | python3_ia |
+| [jsonargparse](https://pypi.org/project/jsonargparse) | 4.46.0 | python3_ia |
+| [jsonnet](https://jsonnet.org) | 0.21.0 | python3_ia |
 | [keras](https://pypi.org/project/keras) | 3.13.2 | python3_ia |
 | [libclang](https://github.com/sighingnow/libclang) | 18.1.1 | python3_ia |
 | [lightning-utilities](https://github.com/Lightning-AI/utilities) | 0.14.3 | python3_ia |
 | [lightning](https://github.com/Lightning-AI/lightning) | 2.5.1.post0 | python3_ia |
 | [lit](http://llvm.org) | 18.1.8 | python3_ia |
+| [marshmallow](https://pypi.org/project/marshmallow) | 3.26.2 | python3_ia |
+| [mfai](https://github.com/meteofrance/mfai) | 6.3.0 | python3_ia |
 | [ml_dtypes](https://github.com/jax-ml/ml_dtypes) | 0.5.1 | python3_ia |
+| [monai](https://monai.io/) | 1.5.2 | python3_ia |
 | [mpmath](http://mpmath.org/) | 1.3.0 | python3_ia |
 | [namex](https://pypi.org/project/namex) | 0.0.9 | python3_ia |
 | [nvidia-cublas-cu12](https://developer.nvidia.com/cuda-zone) | 12.6.4.1 | python3_ia |
@@ -35,6 +46,7 @@
 | [nvidia-nvjitlink-cu12](https://developer.nvidia.com/cuda-zone) | 12.6.85 | python3_ia |
 | [nvidia-nvtx-cu12](https://developer.nvidia.com/cuda-zone) | 12.6.77 | python3_ia |
 | [oauthlib](https://github.com/oauthlib/oauthlib) | 3.2.2 | python3_ia |
+| [omegaconf](https://github.com/omry/omegaconf) | 2.3.0 | python3_ia |
 | [onnx](https://onnx.ai/) | 1.17.0 | python3_ia |
 | [onnxruntime](https://onnxruntime.ai) | 1.22.0 | python3_ia |
 | [onnxscript](https://microsoft.github.io/onnxscript/) | 0.2.5 | python3_ia |
@@ -45,18 +57,23 @@
 | [rsa](https://stuvel.eu/rsa) | 4.9 | python3_ia |
 | [safetensors](https://github.com/huggingface/safetensors) | 0.5.3 | python3_ia |
 | [segmentation_models_pytorch](https://github.com/qubvel-org/segmentation_models.pytorch) | 0.5.0 | python3_ia |
+| [sentencepiece](https://github.com/google/sentencepiece) | 0.2.1 | python3_ia |
 | [sympy](https://sympy.org) | 1.14.0 | python3_ia |
+| [tabulate](https://github.com/astanin/python-tabulate) | 0.9.0 | python3_ia |
 | [tensorboard-data-server](https://github.com/tensorflow/tensorboard/tree/master/tensorboard/data/server) | 0.7.2 | python3_ia |
 | [tensorboard](https://github.com/tensorflow/tensorboard) | 2.20.0 | python3_ia |
 | [tensorflow-hub](https://github.com/tensorflow/hub) | 0.16.1 | python3_ia |
 | [tensorflow](https://www.tensorflow.org/) | 2.20.0 | python3_ia |
 | [tf_keras](https://keras.io/) | 2.20.1 | python3_ia |
+| [tiktoken](https://github.com/openai/tiktoken) | 0.12.0 | python3_ia |
 | [timm](https://github.com/huggingface/pytorch-image-models) | 1.0.15 | python3_ia |
 | [tokenizers](https://github.com/huggingface/tokenizers) | 0.21.1 | python3_ia |
+| [torch-geometric](https://pyg.org) | 2.7.0 | python3_ia |
 | [torch](https://pytorch.org/) | 2.7.1 | python3_ia |
 | [torchmetrics](https://github.com/Lightning-AI/torchmetrics) | 1.7.1 | python3_ia |
 | [torchvision](https://github.com/pytorch/vision) | 0.22.1 | python3_ia |
 | [transformers](https://github.com/huggingface/transformers) | 4.52.1 | python3_ia |
 | [triton](https://github.com/triton-lang/triton/) | 3.3.1 | python3_ia |
+| [typeshed_client](https://github.com/JelleZijlstra/typeshed_client) | 2.8.2 | python3_ia |
 
-*(58 components)*
+*(75 components)*

integration_tests/0003_test_import_python3_ia/import_python3_ia.py

@@ -1,19 +1,30 @@
 import absl #absl_py
+import antlr4 #antlr4-python3-runtime
 import astunparse
+import axial_attention
+import bitsandbytes
+import dataclasses_json
 import diffusers
 import tree #dm_tree
 import einops
 import flatbuffers
 import google_auth_oauthlib
 import google #google_auth
 import huggingface_hub
+import hydra #hydra-core
+import importlib_resources
+import jsonargparse
+import _jsonnet #jsonnet
 import lightning_utilities
 import lightning
 import pasta #google_pasta
 import grpc #grpcio
 import clang #libclang
 import keras
+import marshmallow
+import mfai
 import ml_dtypes
+import monai
 import mpmath
 import namex
 from nvidia import cublas
@@ -30,6 +41,7 @@
 from nvidia import nvjitlink
 from nvidia import nvtx
 import oauthlib
+import omegaconf
 import onnx
 import onnxruntime
 import onnxscript
@@ -40,16 +52,21 @@
 import rsa
 import safetensors
 import segmentation_models_pytorch
+import sentencepiece
 import sympy
+import tabulate
 import tensorboard_data_server
 import tensorboard
 import tensorflow
 import tensorflow_hub
 import tf_keras
+import tiktoken
 import timm
 import tokenizers
 import torch
+import torch_geometric
 import torchmetrics
 import torchvision
 import transformers
 import triton
+import typeshed_client

integration_tests/0096_system_extra_dependencies/list_ok_not_found.txt

@@ -1 +1,20 @@
+libcublas.so.11
+libcublas.so.13
+libcublasLt.so.11
+libcublasLt.so.13
 libcuda.so.1
+libcudart.so.11.0
+libcudart.so.13
+libcusparse.so.11
+libhipblas.so.2
+libhipblas.so.3
+libhipblaslt.so.0
+libhipblaslt.so.1
+libhipsparse.so.1
+libhipsparse.so.4
+libimf.so
+libintlc.so.5
+libirng.so
+libnvJitLink.so.13
+libsvml.so
+libsycl.so.8

layers/layer5_python3_ia/0500_extra_packages/allow_binary_packages

@@ -1,12 +1,15 @@
 absl-py
 aiohappyeyeballs
 astunparse
+bitsandbytes
 cmake
+dataclasses-json
 flatbuffers
 google-auth
 google-auth-oauthlib
 google-pasta
 grpcio
+hydra-core
 Keras
 keras-nightly
 Keras-Preprocessing
@@ -27,6 +30,7 @@ nvidia-nccl-cu12
 nvidia-nvjitlink-cu12
 nvidia-nvtx-cu12
 oauthlib
+omegaconf
 onnx
 onnxruntime
 onnxscript

layers/layer5_python3_ia/0500_extra_packages/requirements-to-freeze.txt

@@ -12,3 +12,4 @@ transformers
 diffusers
 huggingface-hub
 tensorflow-hub
+mfai

layers/layer5_python3_ia/0500_extra_packages/requirements3.txt

@@ -1,5 +1,9 @@
-astunparse==1.6.3
 absl-py==2.2.2
+antlr4_python3_runtime==4.9.3
+astunparse==1.6.3
+axial_attention==0.6.1
+bitsandbytes==0.49.1
+dataclasses_json==0.6.7
 diffusers==0.33.1
 dm-tree==0.1.9
 einops==0.8.1
@@ -9,12 +13,19 @@ google-auth-oauthlib==1.2.2
 google-pasta==0.2.0
 grpcio==1.71.0
 huggingface_hub==0.31.4
+hydra-core==1.3.2
+importlib-resources==6.5.2
+jsonargparse==4.46.0
+jsonnet==0.21.0
 keras==3.13.2
 libclang==18.1.1
 lightning==2.5.1.post0
 lightning-utilities==0.14.3
 lit==18.1.8
+marshmallow==3.26.2
+mfai==6.3.0
 ml-dtypes==0.5.1
+monai==1.5.2
 mpmath==1.3.0
 namex==0.0.9
 nvidia-cublas-cu12==12.6.4.1
@@ -33,6 +44,7 @@ nvidia-nccl-cu12==2.26.2
 nvidia-nvjitlink-cu12==12.6.85
 nvidia-nvtx-cu12==12.6.77
 oauthlib==3.2.2
+omegaconf==2.3.0
 onnx==1.17.0
 onnxruntime==1.22.0
 onnxscript==0.2.5
@@ -43,16 +55,20 @@ requests-oauthlib==2.0.0
 rsa==4.9
 safetensors==0.5.3
 segmentation_models_pytorch==0.5.0
+sentencepiece==0.2.1
 sympy==1.14.0
+tabulate==0.9.0
 tensorboard==2.20.0
 tensorboard-data-server==0.7.2
 tensorflow==2.20.0
 tensorflow-hub==0.16.1
 tf-keras==2.20.1
+tiktoken==0.12.0
 timm==1.0.15
 tokenizers==0.21.1
 torch==2.7.1
 torchmetrics==1.7.1
 torchvision==0.22.1
 transformers==4.52.1
 triton==3.3.1
+typeshed-client==2.8.2