Message-oriented middleware (MOM) is at the core of a vast number of financial services and telco applications, and is gaining increasing traction in other industries, such as manufacturing, transportation, health-care and supply chain management. Novel messaging applications, however, pose some serious performance and scalability challenges. In this paper, we present a methodology for performance evaluation of MOM platforms using the SPECjms2007 benchmark which is the world's first industry-standard benchmark specialized for MOM. SPECjms2007 is based on a novel application in the supply chain management domain designed to stress MOM infrastructures in a manner representative of real-world applications. In addition to providing a standard workload and metrics for MOM performance, the benchmark provides a flexible performance analysis framework that allows users to tailor the workload to their requirements. The contributions of this paper are: i) we present a detailed workload characterization of SPECjms2007 with the goal to help users understand the internal components of the workload and the way they are scaled, ii) we show how the workload can be customized to exercise and evaluate selected aspects of MOM performance, iii) we present a case study of a leading JMS platform, the BEA WebLogic server, conducting an in-depth performance analysis of the platform under a number of different workload and configuration scenarios. The methodology we propose is the first one that uses an industry-standard benchmark providing both a representative workload as well as the ability to customize it to evaluate the features of MOM platforms selectively.
%0 Journal Article
%1 SaKoBaBu2008-PERFEVAL-SPECjms2007
%A Sachs, Kai
%A Kounev, Samuel
%A Bacon, Jean
%A Buchmann, Alejandro
%C Amsterdam, The Netherlands
%D 2009
%I Elsevier Science Publishers B. V.
%J Performance Evaluation
%K Event-based Instrumentation_profiling_and_workload_characterization Metrics_and_benchmarking_methodologies Performance SPEC Tool descartes t_journalmagazine
%N 8
%P 410--434
%T Performance evaluation of message-oriented middleware using the SPECjms2007 benchmark
%U http://www.elsevier.com/wps/find/journaldescription.cws_home/505618/description
%V 66
%X Message-oriented middleware (MOM) is at the core of a vast number of financial services and telco applications, and is gaining increasing traction in other industries, such as manufacturing, transportation, health-care and supply chain management. Novel messaging applications, however, pose some serious performance and scalability challenges. In this paper, we present a methodology for performance evaluation of MOM platforms using the SPECjms2007 benchmark which is the world's first industry-standard benchmark specialized for MOM. SPECjms2007 is based on a novel application in the supply chain management domain designed to stress MOM infrastructures in a manner representative of real-world applications. In addition to providing a standard workload and metrics for MOM performance, the benchmark provides a flexible performance analysis framework that allows users to tailor the workload to their requirements. The contributions of this paper are: i) we present a detailed workload characterization of SPECjms2007 with the goal to help users understand the internal components of the workload and the way they are scaled, ii) we show how the workload can be customized to exercise and evaluate selected aspects of MOM performance, iii) we present a case study of a leading JMS platform, the BEA WebLogic server, conducting an in-depth performance analysis of the platform under a number of different workload and configuration scenarios. The methodology we propose is the first one that uses an industry-standard benchmark providing both a representative workload as well as the ability to customize it to evaluate the features of MOM platforms selectively.
@article{SaKoBaBu2008-PERFEVAL-SPECjms2007,
abstract = {Message-oriented middleware (MOM) is at the core of a vast number of financial services and telco applications, and is gaining increasing traction in other industries, such as manufacturing, transportation, health-care and supply chain management. Novel messaging applications, however, pose some serious performance and scalability challenges. In this paper, we present a methodology for performance evaluation of MOM platforms using the SPECjms2007 benchmark which is the world's first industry-standard benchmark specialized for MOM. SPECjms2007 is based on a novel application in the supply chain management domain designed to stress MOM infrastructures in a manner representative of real-world applications. In addition to providing a standard workload and metrics for MOM performance, the benchmark provides a flexible performance analysis framework that allows users to tailor the workload to their requirements. The contributions of this paper are: i) we present a detailed workload characterization of SPECjms2007 with the goal to help users understand the internal components of the workload and the way they are scaled, ii) we show how the workload can be customized to exercise and evaluate selected aspects of MOM performance, iii) we present a case study of a leading JMS platform, the BEA WebLogic server, conducting an in-depth performance analysis of the platform under a number of different workload and configuration scenarios. The methodology we propose is the first one that uses an industry-standard benchmark providing both a representative workload as well as the ability to customize it to evaluate the features of MOM platforms selectively.},
added-at = {2020-04-05T23:14:48.000+0200},
address = {Amsterdam, The Netherlands},
author = {Sachs, Kai and Kounev, Samuel and Bacon, Jean and Buchmann, Alejandro},
biburl = {https://www.bibsonomy.org/bibtex/29d581b8f17fcef92485096693c83e879/se-group},
interhash = {826b41563874bf999b1ceff038516a6e},
intrahash = {9d581b8f17fcef92485096693c83e879},
journal = {Performance Evaluation},
keywords = {Event-based Instrumentation_profiling_and_workload_characterization Metrics_and_benchmarking_methodologies Performance SPEC Tool descartes t_journalmagazine},
month = {August},
number = 8,
pages = {410--434},
publisher = {Elsevier Science Publishers B. V.},
timestamp = {2020-10-01T19:19:12.000+0200},
title = {{Performance evaluation of message-oriented middleware using the SPECjms2007 benchmark}},
url = {http://www.elsevier.com/wps/find/journaldescription.cws_home/505618/description},
volume = 66,
year = 2009
}