Marcos Gomez-Vazquez and Jordi Cabot
[pdf]
Best Demonstrations and Tools Paper Award
28th ACM International Systems and Software Product Line Conference (SPLC 2024)
Dommeldange, Luxembourg
https://doi.org/10.1145/3646548.3676599
The size of Large Language Models (LLMs), and Machine Learning (ML) models in general, is a key factor of their capacity and quality of their responses. But it comes with a high cost, both during the training and the model execution phase. Recently, various model merging techniques and Mixture of Experts (MoE) architectures are gaining popularity as they enable the creation of large models by combining other existing ones (the "experts" in the MoE approach). Creating these combinations remains a deep technical task with many possible configurations to consider. In this sense, this paper aims to democratize the creation of combined ML models by presenting a product line approach to the specification and training of this type of ML architectures from an initial feature model that helps users define, among other aspects, the type of models they want to combine, the combination strategy and even, for the MoE approach, the tasks that should be associated to each expert.
Software Product Line, Feature Model, Machine Learning, Large Language Model, Model Merging, Mixture of Experts
@inproceedings{10.1145/3646548.3676599,
author = {Gomez-Vazquez, Marcos and Cabot, Jordi},
title = {Exploring the Use of Software Product Lines for the Combination of Machine Learning Models},
year = {2024},
isbn = {9798400705939},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3646548.3676599},
doi = {10.1145/3646548.3676599},
abstract = {The size of Large Language Models (LLMs), and Machine Learning (ML) models in general, is a key factor of their capacity and quality of their responses. But it comes with a high cost, both during the training and the model execution phase. Recently, various model merging techniques and Mixture of Experts (MoE) architectures are gaining popularity as they enable the creation of large models by combining other existing ones (the "experts" in the MoE approach). Creating these combinations remains a deep technical task with many possible configurations to consider. In this sense, this paper aims to democratize the creation of combined ML models by presenting a product line approach to the specification and training of this type of ML architectures from an initial feature model that helps users define, among other aspects, the type of models they want to combine, the combination strategy and even, for the MoE approach, the tasks that should be associated to each expert.},
booktitle = {Proceedings of the 28th ACM International Systems and Software Product Line Conference},
pages = {26–29},
numpages = {4},
keywords = {Feature Model, Large Language Model, Machine Learning, Mixture of Experts, Model Merging, Software Product Line},
location = {Dommeldange, Luxembourg},
series = {SPLC '24}
}