SymbioticLab.bib

% Encoding: UTF-8
@InProceedings{hug:nsdi16,
  Title                    = {{HUG}: Multi-Resource Fairness for Correlated and Elastic Demands},
  Author                   = {Mosharaf Chowdhury and Zhenhua Liu and Ali Ghodsi and Ion Stoica},
  Booktitle                = {USENIX NSDI},
  Year                     = {2016},
  Month                    = {March},

  publist_confkey = {NSDI'16},
  publist_link = {paper || hug-nsdi16.pdf},
  publist_topic = {Datacenter Networking},
  Abstract                 = {In this paper, we study how to optimally provide isolation guarantees in multi-resource environments, such as public clouds, where a tenant's demands on different resources (links) are correlated. Unlike prior work such as Dominant Resource Fairness (DRF) that assumes static and fixed demands, we consider elastic demands. Our approach generalizes canonical max-min fairness to the multi-resource setting with correlated demands, and extends DRF to elastic demands. We consider two natural optimization objectives: isolation guarantee from a tenant's viewpoint and system utilization (work conservation) from an operator's perspective. We prove that in non-cooperative environments like public cloud networks, there is a strong tradeoff between optimal isolation guarantee and work conservation when demands are elastic. Even worse, work conservation can even decrease network utilization instead of improving it when demands are inelastic. We identify the root cause behind the tradeoff and present a provably optimal allocation algorithm, High Utilization with Guarantees (HUG), to achieve maximum attainable network utilization without sacrificing the optimal isolation guarantee, strategy-proofness, and other useful properties of DRF. In cooperative environments like private datacenter networks, HUG achieves both the optimal isolation guarantee and work conservation. Analyses, simulations, and experiments show that HUG provides better isolation guarantees, higher system utilization, and better tenant-level performance than its counterparts.}
}

@InProceedings{carbyne:osdi16,
  Title                    = {Altruistic Scheduling in Multi-Resource Clusters},
  Author                   = {Robert Grandl and Mosharaf Chowdhury and Aditya Akella and Ganesh Ananthanarayanan},
  Booktitle                = {USENIX OSDI},
  Year                     = {2016},
  Month                    = {October},

  publist_confkey = {OSDI'16},
  publist_link = {paper || carbyne-osdi16.pdf},
  publist_topic = {Big Data Systems},
  Abstract                 = {Given the well-known tradeoffs between fairness, performance, and efficiency, modern cluster schedulers often prefer instantaneous fairness as their primary objective to ensure performance isolation between users and groups. However, instantaneous, short-term convergence to fairness often does not result in noticeable long-term benefits. Instead, we propose an altruistic, long-term approach, Carbyne, where jobs yield fractions of their allocated resources without impacting their own completion times. We show that leftover resources collected via altruisms of many jobs can then be rescheduled to further secondary goals such as application-level performance and cluster efficiency without impacting performance isolation. Deployments and large-scale simulations show that Carbyne closely approximates the state-of-the-art solutions (e.g., DRF) in terms of performance isolation, while providing 1.26X better efficiency and 1.59X lower average job completion time.}
}

@Article{infiniswap:login17,
  Title                    = {Decentralized Memory Disaggregation Over Low-Latency Networks},
  Author                   = {Juncheng Gu and Youngmoon Lee and Yiwen Zhang and Mosharaf Chowdhury and Kang G. Shin},
  Journal                  = {USENIX ;login:},
  Year                     = {2017},

  Month                    = {December},
  Number                   = {4},
  Pages                    = {42--48},
  Volume                   = {42},
  publist_confkey = {USENIX ;login: Winter 2017},
  publist_link = {paper || infiniswap-login17.pdf},
  publist_topic = {Disaggregation},
  Abstract                 = {Memory disaggregation can expose remote memory across a cluster to local applications. However, existing proposals call for new architectures and/or new programming models, making them infeasible. We have developed a practical memory disaggregation solution, Infiniswap, which is a remote memory paging system for clusters with lowlatency, kernel-bypass networks such as RDMA. Infiniswap opportunistically harvests and transparently exposes unused memory across the cluster to unmodified applications by dividing the swap space of each machine into many chunks and distributing them to unused memory of many remote machines. For scalability, it leverages the power of many choices to perform decentralized memory chunk placements and evictions. Applications using Infiniswap receive large performance boosts when their working sets are larger than their physical memory allocations.}
}

@InProceedings{infiniswap:nsdi17,
  Title                    = {Efficient Memory Disaggregation with {Infiniswap}},
  Author                   = {Juncheng Gu and Youngmoon Lee and Yiwen Zhang and Mosharaf Chowdhury and Kang G. Shin},
  Booktitle                = {USENIX NSDI},
  Year                     = {2017},
  Month                    = {March},

  publist_confkey = {NSDI'17},
  publist_link = {paper || infiniswap-nsdi17.pdf},
  publist_link = {slides || infiniswap-nsdi17-slides.pdf},
  publist_link = {code || https://github.com/SymbioticLab/Infiniswap},
  publist_link = {website || https://infiniswap.github.io/},
  publist_link = {media || https://blog.acolyer.org/2017/05/05/efficient-memory-disaggregation-with-infiniswap/},
  publist_topic = {Disaggregation},
  publist_abstract = {
  Memory-intensive applications suffer large performance loss when their working sets do not fully fit in memory. Yet, they cannot leverage otherwise unused remote memory when paging out to disks even in the presence of large imbalance in memory utilizations across a cluster. Existing proposals for memory disaggregation call for new architectures, new hardware designs, and/or new programming models, making them infeasible.
  This paper describes the design and implementation of Infiniswap, a remote memory paging system designed specifically for an RDMA network. Infiniswap opportunistically harvests and transparently exposes unused memory to unmodified applications by dividing the swap space of each machine into many slabs and distributing them across many machines' remote memory. Because one-sided RDMA operations bypass remote CPUs, Infiniswap leverages the power of many choices to perform decentralized slab placements and evictions.
  We have implemented and deployed Infiniswap on an RDMA cluster without any modifications to user applications or the OS and evaluated its effectiveness using multiple workloads running on unmodified VoltDB, Memcached, PowerGraph, GraphX, and Apache Spark. Using Infiniswap, throughputs of these applications improve between 4X (0.94X) to 15.4X (7.8X) over disk (Mellanox nbdX), and median and tail latencies between 5.4X (2X) and 61X (2.3X). Infiniswap achieves these with negligible remote CPU usage, whereas nbdX becomes CPU-bound. Infiniswap increases the overall memory utilization of a cluster and works well at scale.}
  }

@TechReport{cellscope:tr16,
  Title                    = {Fast and Accurate Performance Analysis of {LTE} Radio Access Networks},
  Author                   = {Anand Padmanabha Iyer and Ion Stoica and Mosharaf Chowdhury and Li Erran Li},
  Institution              = {CoRR},
  Year                     = {2016},
  Month                    = {May},
  Number                   = {abs/1605.04652},

  publist_confkey = {arXiv:1605.04652},
  publist_link = {paper || https://arxiv.org/abs/1605.04652},
  publist_topic = {Wide-Area Computing},
  Abstract                 = {An increasing amount of analytics is performed on data that is procured in a real-time fashion to make real-time decisions. Such tasks include simple reporting on streams to sophisticated model building. However, the practicality of such analyses are impeded in several domains because they are faced with a fundamental trade-off between data collection latency and analysis accuracy.

In this paper, we study this trade-off in the context of a specific domain, Cellular Radio Access Networks (RAN). Our choice of this domain is influenced by its commonalities with several other domains that produce real-time data, our access to a large live dataset, and their real-time nature and dimensionality which makes it a natural fit for a popular analysis technique, machine learning (ML). We find that the latency accuracy trade-off can be resolved using two broad, general techniques: intelligent data grouping and task formulations that leverage domain characteristics. Based on this, we present CellScope, a system that addresses this challenge by applying a domain specific formulation and application of Multi-task Learning (MTL) to RAN performance analysis. It achieves this goal using three techniques: feature engineering to transform raw data into effective features, a PCA inspired similarity metric to group data from geographically nearby base stations sharing performance commonalities, and a hybrid online-offline model for efficient model updates. Our evaluation of CellScope shows that its accuracy improvements over direct application of ML range from 2.5x to 4.4x while reducing the model update overhead by up to 4.8x. We have also used CellScope to analyze a live LTE consisting of over 2 million subscribers for a period of over 10 months, where it uncovered several problems and insights, some of them previously unknown.}
}

@InProceedings{deepstack:hotos17,
  Title                    = {No! Not Another Deep Learning Framework},
  Author                   = {Linh Nguyen and Peifeng Yu and Mosharaf Chowdhury},
  Booktitle                = {ACM HotOS},
  Year                     = {2017},
  Month                    = {May},

  publist_confkey = {HotOS'17},
  publist_link = {paper || deepstack-hotos17.pdf},
  publist_topic = {Systems + AI},
  publist_abstract = {
    In recent years, deep learning has pervaded many areas of computing due to the confluence of an explosive growth of large-scale computing capabilities, availability of datasets, and advances in learning techniques.
    While this rapid growth has resulted in diverse deep learning frameworks, it has also led to inefficiencies for both the users and developers of these frameworks.
    Specifically, adopting useful techniques across frameworks -- both to perform learning tasks and to optimize performance -- involves significant repetitions and reinventions.

    In this paper, we observe that despite their diverse origins, many of these frameworks share architectural similarities.
    We argue that by introducing a common representation of learning tasks and a hardware abstraction model to capture compute heterogeneity, we might be able to relieve machine learning researchers from dealing with low-level systems issues and systems researchers from being tied to any specific framework.
    We expect this decoupling to accelerate progress in both domains.
  }
}

@InProceedings{eccache:osdi16,
  Title                    = {{EC-Cache}: Load-Balanced, Low-Latency Cluster Caching with Online Erasure Coding},
  Author                   = {K. V. Rashmi and Mosharaf Chowdhury and Jack Kosaian and Ion Stoica and Kannan Ramchandran},
  Booktitle                = {USENIX OSDI},
  Year                     = {2016},
  Month                    = {October},

  publist_confkey = {OSDI'16},
  publist_link = {paper || eccache-osdi16.pdf},
  publist_topic = {Big Data Systems},
  Abstract                 = {Data-intensive clusters and object stores are increasingly relying on in-memory object caching to meet the I/O performance demands. These systems routinely face the challenges of popularity skew, background load imbalance, and server failures, which result in severe load imbalance across storage servers and degraded I/O performance. Selective replication is a commonly used technique to tackle these challenges, where the number of cached replicas of an object is proportional to its popularity. In this paper, we explore an alternative approach using erasure coding.

EC-Cache is a load-balanced, low latency cluster cache that uses online erasure coding to overcome the limitations of selective replication. EC-Cache employs erasure coding by: (i) splitting and erasure coding individual objects during writes, and (ii) late binding, wherein obtaining any k out of (k + r) splits of an object are sufficient, during reads. As compared to selective replication, EC-Cache improves load balancing by more than 3X and reduces the median and tail read latencies by more than 2X, while using the same amount of memory. EC-Cache does so using 10% additional bandwidth and a small increase in the amount of stored metadata. The benefits offered by EC-Cache are further amplified in the presence of background network load imbalance and server failures.}
}

@InProceedings{coda:sigcomm16,
  Title                    = {{CODA}: Toward Automatically Identifying and Scheduling {CO}flows in the {DA}rk},
  Author                   = {Hong Zhang and Li Chen and Bairen Yi and Kai Chen and Mosharaf Chowdhury and Yanhui Geng},
  Booktitle                = {ACM SIGCOMM},
  Year                     = {2016},
  Month                    = {August},

  publist_confkey = {SIGCOMM'16},
  publist_link = {paper || coda-sigcomm16.pdf},
  publist_topic = {Datacenter Networking},  
  Abstract                 = {Leveraging application-level requirements using coflows has recently been shown to improve application-level communication performance in data-parallel clusters. However, existing coflow-based solutions rely on modifying applications to extract coflows, making them inapplicable to many practical scenarios.

In this paper, we present CODA, a first attempt at automatically identifying and scheduling coflows without any application modifications. We employ an incremental clustering algorithm to perform fast, application-transparent coflow identification and complement it by proposing an error-tolerant coflow scheduler to mitigate occasional identification errors. Testbed experiments and large-scale simulations with production workloads show that CODA can identify coflows with over 90% accuracy, and its scheduler is robust to inaccuracies, enabling communication stages to complete 2.4X (5.1X) faster on average (95th percentile) compared to per-flow mechanisms. Overall, CODA's performance is comparable to that of solutions requiring application modifications.}
}

@InProceedings{hermes:sigcomm17,
  Title                    = {Resilient Datacenter Load Balancing in the Wild},
  Author                   = {Hong Zhang and Junxue Zhang and Wei Bai and Kai Chen and Mosharaf Chowdhury},
  Booktitle                = {ACM SIGCOMM},
  Year                     = {2017},
  Month                    = {August},

  publist_confkey = {SIGCOMM'17},
  publist_link = {paper || hermes-sigcomm17.pdf},
  publist_topic = {Datacenter Networking},
  Abstract                 = {Production datacenters operate under various uncertainties such as traffic dynamics, topology asymmetry, and failures. Therefore, datacenter load balancing schemes must be resilient to these uncertainties; i.e., they should accurately sense path conditions and timely react to mitigate the fallouts. Despite significant efforts, prior solutions have important drawbacks. On the one hand, solutions such as Presto and DRB are oblivious to path conditions and blindly reroute at fixed granularity. On the other hand, solutions such as CONGA and CLOVE can sense congestion, but they can only reroute when flowlets emerge; thus, they cannot always react timely to uncertainties. To make things worse, these solutions fail to detect/handle failures such as blackholes and random packet drops, which greatly degrades their performance.

In this paper, we introduce Hermes, a datacenter load balancer that is resilient to the aforementioned uncertainties. At its heart, Hermes leverages comprehensive sensing to detect path conditions including failures unattended before, and it reacts using timely yet cautious rerouting. Hermes is a practical edge-based solution with no switch modification. We have implemented Hermes with commodity switches and evaluated it through both testbed experiments and large-scale simulations. Our results show that Hermes achieves comparable performance to CONGA and Presto in normal cases, and well handles uncertainties: under asymmetries, Hermes achieves up to 10% and 20% better flow completion time (FCT) than CONGA and CLOVE; under switch failures, it outperforms all other schemes by over 32%.}
}

@InProceedings{frdma:kbnets17,
  Title                    = {Performance Isolation Anomalies in {RDMA}},
  Author                   = {Yiwen Zhang and Juncheng Gu and Youngmoon Lee and Mosharaf Chowdhury and Kang G. Shin},
  Booktitle                = {ACM SIGCOMMKBNets},
  Year                     = {2017},
  Month                    = {August},

  publist_confkey          = {KBNets'17},
  publist_link             = {paper || frdma-kbnets17.pdf},
  publist_link             = {slides || frdma-kbnets17-slides.pdf},
  publist_topic = {Datacenter Networking},
  Abstract                 = {To meet the increasing throughput and latency demands of modern applications, many operators are rapidly deploying RDMA in their datacenters. At the same time, developers are re-designing their software to take advantage of RDMA's benefits for individual applications. However, when it comes to RDMA's performance, many simple questions remain open.

In this paper, we consider the performance isolation characteristics of RDMA. Specifically, we conduct three sets of experiments -- three combinations of one throughput-sensitive flow and one latency-sensitive flow -- in a controlled environment, observe large discrepancies in RDMA performance with and without the presence of a competing flow, and describe our progress in identifying plausible root-causes.}
}

@InProceedings{pdd:apnet18,
  author    = {Hong Zhang and Kai Chen and Mosharaf Chowdhury},
  booktitle = {ACM APNet},
  title     = {Pas de deux: Shape the Circuits, and Shape the Apps too!},
  year      = {2018},
  pages     = {29--35},

  publist_confkey = {APNet'18},
  publist_link = {paper || pdd-apnet18.pdf},
  publist_topic = {Datacenter Networking},
  Abstract                 = {Despite continued efforts toward building high bandwidth, low cost datacenter networks with reconfigurable optical fabrics, the impact of optical networks on datacenter applications has received little attention. Given the constraints of optical networks and the semantics of datacenter applications, we believe the network-application intersection to be the next innovation hotspot. In this paper, we specifically focus on data-parallel applications for two primary reasons: they are a natural fit to exploit high bandwidth optical fabrics, and they often form structured communication patterns or coflows.

    We show that configuring circuits in reaction to changing traffic patterns is not enough. Efficient scheduling of even a single coflow in optical networks should be a "Pas de deux" – a joint shaping of not only the underlying circuit, but also the application’s traffic demand. Our preliminary evaluation with a production trace shows that joint shaping is on average within 1.18X of the optimal and performs 30% better than solu- tions that configure circuits in application-agnostic fashions. We further extend our analysis to inter-coflow scheduling and propose a layered solution that jointly considers circuit reconfiguration, coflow prioritization, as well as flow rate and route assignments.}
}

@InProceedings{aga:grades-nda18,
  author    = {Anand Padmanabha Iyer and Aurojit Panda and Shivaram Venkataraman and Mosharaf Chowdhury and Aditya Akella and Scott Shenker and Ion Stoica},
  booktitle = {ACM SIGMOD GRADES-NDA},
  title     = {Bridging the {GAP}: Towards Approximate Graph analytics},
  year      = {2018},

  publist_confkey = {GRADES-NDA'18},
  publist_link = {paper || aga-grades-nda18.pdf},
  publist_badge = {Best Paper Award},
  publist_topic = {Big Data Systems},
  Abstract                 = {While there has been a tremendous interest in processing data that has an underlying graph structure, existing distributed graph processing systems take several minutes or even hours to execute popular graph algorithms. However, in several cases, providing an approximate answer is good enough. Approximate analytics is seeing considerable attention in big data due to its ability to produce timely results by trading accuracy, but they do not support graph analytics. In this paper, we bridge this gap and take a first attempt at realizing approximate graph analytics. We discuss how traditional approximate analytics techniques do not carry over to the graph usecase. Leveraging the characteristics of graph properties and algorithms, we propose a graph sparsification technique, and a machine learning based approach to choose the apt amount of sparsification required to meet a given budget. Our preliminary evaluations show encouraging results.}
}

@InProceedings{monarch:hotcloud18,
  author    = {Anand Padmanabha Iyer and Aurojit Panda and Mosharaf Chowdhury and Aditya Akella and Scott Shenker and Ion Stoica},
  booktitle = {USENIX HotCloud},
  title     = {Monarch: Gaining Command on Geo-Distributed Graph Analytics},
  year      = {2018},

  publist_confkey = {HotCloud'18},
  publist_link = {paper || monarch-hotcloud18.pdf},
  publist_topic = {Wide-Area Computing},
  Abstract                 = {A number of existing and emerging application scenarios generate graph-structured data in a geo-distributed fashion. Although there is a lot of interest in distributed graph processing systems, none of them support geo-distributed graph processing. Geo-distributed analytics, on the other hand, has not focused on iterative workloads such as distributed graph processing.

    In this paper, we look at the problem of efficient geo-distributed graph analytics. We find that optimizing the iterative processing style of graph-parallel systems is the key to achieving this goal rather than extending existing geo-distributed techniques to graph processing. Based on this, we discuss our proposal on building Monarch, the first system to our knowledge that focuses on geo-distributed graph processing. Our preliminary evaluation of Monarch shows encouraging results.}
}

@InProceedings{relay:hotcloud18,
  author    = {Fan Lai and Mosharaf Chowdhury and Harsha V. Madhyastha},
  booktitle = {USENIX HotCloud},
  title     = {To Relay or Not to Relay for Inter-Cloud Transfers?},
  year      = {2018},

  publist_confkey = {HotCloud'18},
  publist_link = {paper || relay-hotcloud18.pdf},
  publist_link = {slides || relay-hotcloud18-slides.pdf},
  publist_topic = {Wide-Area Computing},
  Abstract    = {
    Efficient big data analytics over the wide-area network (WAN) is becoming increasingly more popular. Current geo-distributed analytics (GDA) systems employ WANaware optimizations to tackle WAN heterogeneities. Although extensive measurements on public clouds suggest the potential for improving inter-datacenter data transfers via detours, we show that such optimizations are unlikely to work in practice. This is because the widely accepted mantra used in a large body of literature – WAN bandwidth has high variability – can be misleading. Instead, our measurements across 40 datacenters belonging to Amazon EC2, Microsoft Azure, and Google Cloud Platform show that the available WAN bandwidth is often spatially homogeneous and temporally stable between two virtual machines (VMs) in different datacenters, even though it can be heterogeneous at the TCP flow level. Moreover, there is little scope for either bandwidth or latency optimization in a cost-effective manner via relaying. We believe that these findings will motivate the community to rethink the design rationales of GDA systems and geo-distributed services.
  }
}

@InProceedings{cellscope:mobicom18,
  author    = {Anand Padmanabha Iyer and Li Erran Li and Mosharaf Chowdhury and Ion Stoica},
  booktitle = {ACM MobiCom},
  title     = {Mitigating the Latency-Accuracy Trade-off in Mobile Data Analytics Systems},
  year      = {2018},
  pages     = {513--528},

  publist_confkey = {MobiCom'18},
  publist_link = {paper || cellscope-mobicom18.pdf},
  publist_topic = {Wide-Area Computing},
  Abstract                 = {An increasing amount of mobile analytics is performed on data that is procured in a real-time fashion to make real-time decisions. Such tasks include simple reporting on streams to sophisticated model building. However, the practicality of these analyses are impeded in several domains because they are faced with a fundamental trade-off between data collection latency and analysis accuracy.

    In this paper, we first study this trade-off in the context of a specific domain, Cellular Radio Access Networks (RAN). We find that the trade-off can be resolved using two broad, general techniques: intelligent data grouping and task formulations that leverage domain characteristics. Based on this, we present CellScope, a system that applies a domain specific formulation and application of Multi-task Learning (MTL) to RAN performance analysis. It uses three techniques: feature engineering to transform raw data into effective features, a PCA inspired similarity metric to group data from geographically nearby base stations sharing performance commonalities, and a hybrid online-offline model for efficient model updates. Our evaluation shows that CellScope's accuracy improvements over direct application of ML range from 2.5X to 4.4X while reducing the model update overhead by up to 4.8X. We have also used CellScope to analyze an LTE network of over 2 million subscribers, where it reduced troubleshooting efforts by several magnitudes.}
}

@InProceedings{qoop:osdi18,
  author    = {Kshiteej Mahajan and Mosharaf Chowdhury and Aditya Akella and Shuchi Chawla},
  booktitle = {USENIX OSDI},
  title     = {Dynamic Query Re-Planning using {QOOP}},
  year      = {2018},
  pages     = {253--267},

  publist_confkey = {OSDI'18},
  publist_link = {paper || qoop-osdi18.pdf},
  publist_topic = {Big Data Systems},
  Abstract                 = {Modern data processing clusters are highly dynamic – both in terms of the number of concurrently running jobs and their resource usage. To improve job performance, recent works have focused on optimizing the cluster scheduler and the jobs' query planner with a focus on picking the right query execution plan (QEP) – represented as a directed acyclic graph – for a job in a resource-aware manner, and scheduling jobs in a QEP-aware manner. However, because existing solutions use a fixed QEP throughout the entire execution, the inability to adapt a QEP in reaction to resource changes often leads to large performance inefficiencies.

    This paper argues for dynamic query re-planning, wherein we re-evaluate and re-plan a job's QEP during its execution. We show that designing for re-planning requires fundamental changes to the interfaces between key layers of data analytics stacks today, i.e., the query planner, the execution engine, and the cluster scheduler. Instead of pushing more complexity into the scheduler or the query planner, we argue for a redistribution of responsibilities between the three components to simplify their designs. Under this redesign, we analytically show that a greedy algorithm for re-planning and execution alongside a simple max-min fair scheduler can offer provably competitive behavior even under adversarial resource changes. We prototype our algorithms atop Apache Hive and Tez. Via extensive experiments, we show that our design can offer a median performance improvement of 1.47X compared to state-of-the-art alternatives.}
}

@InProceedings{dslr:sigmod18,
  author    = {Dong Young Yoon and Mosharaf Chowdhury and Barzan Mozafari},
  booktitle = {ACM SIGMOD},
  title     = {Distributed Lock Management with {RDMA}: Decentralization without Starvation},
  year      = {2018},
  pages     = {1571--1586},

  publist_confkey = {SIGMOD'18},
  publist_link = {paper || dslr-sigmod18.pdf},
  publist_topic = {Datacenter Networking},
  publist_topic = {Disaggregation},
  Abstract                 = {Lock managers are a crucial component of modern distributed systems. However, with the increasing availability of fast RDMA-enabled networks, traditional lock managers can no longer keep up with the latency and throughput requirements of modern systems. Centralized lock managers can ensure fairness and prevent starvation using global knowledge of the system, but are themselves single points of contention and failure. Consequently, they fall short in leveraging the full potential of RDMA networks. On the other hand, decentralized (RDMA-based) lock managers either completely sacrifice global knowledge to achieve higher throughput at the risk of starvation and higher tail latencies, or they resort to costly communications in order to maintain global knowledge, which can result in significantly lower throughput.
  
    In this paper, we show that it is possible for a lock manager to be fully decentralized and yet exchange the partial knowledge necessary for preventing starvation and thereby reducing tail latencies. Our main observation is that we can design a lock manager primarily using RDMA's fetch-and-add (FA) operations, which always succeed, rather than compare-and-swap (CAS) operations, which only succeed if a given condition is satisfied. While this requires us to rethink the locking mechanism from the ground up, it enables us to sidestep the performance drawbacks of the previous CAS-based proposals that relied solely on blind retries upon lock conflicts.

    Specifically, we present DSLR (Decentralized and Starvation-free Lock management with RDMA), a decentralized lock manager that targets distributed systems running on RDMA-enabled networks. We demonstrate that, despite being fully decentralized, DSLR prevents starvation and blind retries by guaranteeing first-come-first-serve (FCFS) scheduling without maintaining explicit queues. We adapt Lamport's bakery algorithm [36] to an RDMA-enabled environment with multiple bakers, utilizing only one-sided READ and atomic FA operations. Our experiments show that, on average, DSLR delivers 1.8X (and up to 2.8X) higher throughput than all existing RDMA-based lock managers, while reducing their mean and 99.9% latencies by 2.0X and 18.3X (and up to 2.5X and 47X), respectively.}
}

@InProceedings{allox:mama18,
  author    = {Xiao Sun and Tan N. Le and Mosharaf Chowdhury and Zhenhua Liu},
  booktitle = {ACM SIGMETRICS MAMA},
  title     = {Fair Allocation of Heterogeneous and Interchangeable Resources},
  year      = {2018},

  publist_confkey = {MAMA'18},
  publist_link = {paper || allox-mama18.pdf},
  publist_topic = {Systems + AI},
  Abstract                 = {Motivated by the proliferation of heterogeneous processors such as multi-core CPUs, GPUs, TPUs, and other accelerators for machine learning, we formulate a novel multi-interchangeable resource allocation (MIRA) problem where some resources are interchangeable. The challenge is how to allocate interchangeable resources to users in a sharing system while maintaining desirable properties such as sharing incentive, Pareto efficiency, and envy-freeness. In this paper, we first show that existing algorithms, including the Dominant Resource Fairness used in production systems, fail to provide these properties for interchangeable resources. Then we characterize the tradeoff between performance and strategyproofness, and design the Budget-based (BUD) algorithm, which preserves Pareto efficiency, sharing incentive and envy-freeness while providing better performance over currently used algorithms.
}
}

@InProceedings{tiresias:nsdi19,
  author    = {Juncheng Gu and Mosharaf Chowdhury and Kang G. Shin and Yibo Zhu and Myeongjae Jeon and Junjie Qian and Hongqiang Harry Liu and Chuanxiong Guo},
  booktitle = {USENIX NSDI},
  title     = {Tiresias: A {GPU} Cluster Manager for Distributed Deep Learning},
  year      = {2019},
  pages     = {485--500},

  publist_confkey = {NSDI'19},
  publist_link = {paper || tiresias-nsdi19.pdf},
  publist_link = {slides || tiresias-nsdi19-slides.pdf},
  publist_link = {pps || tiresias-nsdi19-slides.ppsx},
  publist_link = {code || https://github.com/SymbioticLab/Tiresias},
  publist_topic = {Systems + AI},
  publist_abstract = {
  Deep learning (DL) training jobs bring some unique challenges to existing cluster managers, such as unpredictable training times, an all-or-nothing execution model, and inflexibility in GPU sharing. Our analysis of a large GPU cluster in production shows that existing big data schedulers cause long queueing delays and low overall performance.
  
  We present Tiresias, a GPU cluster manager tailored for distributed DL training jobs, which efficiently schedules and places DL jobs to reduce their job completion times (JCTs). Given that a DL job’s execution time is often unpredictable, we propose two scheduling algorithms – Discretized Two-Dimensional Gittins index relies on partial information and Discretized Two-Dimensional LAS is information-agnostic – that aim to minimize the average JCT. Additionally, we describe when the consolidated placement constraint can be relaxed, and present a placement algorithm to leverage these observations without any user input. Experiments on the Michigan ConFlux cluster with 60 P100 GPUs and large-scale trace-driven simulations show that Tiresias improves the average JCT by up to 5.5× over an Apache YARN-based resource manager used in production. More importantly, Tiresias’s performance is comparable to that of solutions assuming perfect knowledge.}
}

@InProceedings{nocs:spaa19,
  author    = {Mosharaf Chowdhury and Samir Khuller and Manish Purohit and Sheng Yang and Jie You},
  booktitle = {ACM SPAA},
  title     = {Near Optimal Coflow Scheduling in Networks},
  year      = {2019},
  pages     = {123--134},

  publist_confkey = {SPAA'19},
  publist_link = {paper || nocs-spaa19.pdf},
  publist_topic = {Wide-Area Computing},
  publist_abstract  = {The coflow scheduling problem has emerged as a popular abstraction in the last few years to study data communication problems within a data center. In this basic framework, each coflow has a set of communication demands and the goal is to schedule many coflows in a manner that minimizes the total weighted completion time. A coflow is said to complete when all its communication needs are met. This problem has been extremely well studied for the case of complete bipartite graphs that model a data center with full bisection bandwidth and several approximation algorithms and effective heuristics have been proposed recently. In this work, we study a slightly different model of coflow scheduling in general graphs (to capture traffic between data centers) and develop practical and efficient approximation algorithms for it. Our main result is a randomized 2 approximation algorithm for the single path and free path model, significantly improving prior work. In addition, we demonstrate via extensive experiments that the algorithm is practical, easy to implement and performs well in practice.
  }
}

@Article{memtrade:arxiv21,
  author        = {Hasan Al Maruf and Yuhong Zhong and Hongyi Wang and Mosharaf Chowdhury and Asaf Cidon and Carl Waldspurger},
  journal       = {CoRR},
  title         = {Memtrade: A Disaggregated-Memory Marketplace for Public Clouds},
  year          = {2021},
  month         = {Aug},
  volume        = {abs/2108.06893},
  archiveprefix = {arXiv},
  eprint        = {2108.06893},
  url           = {https://arxiv.org/abs/2108.06893},
  publist_confkey = {arXiv:2108.06893},
  publist_link = {paper || https://arxiv.org/abs/2108.06893},
  publist_topic = {Disaggregation},
  publist_abstract = {
    We present Memtrade, the first memory disaggregation system for public clouds. Public clouds introduce a set of unique challenges for resource disaggregation across different tenants, including security, isolation and pricing. Memtrade allows producer virtual machines (VMs) to lease both their unallocated memory and allocated-but-idle application memory to remote consumer VMs for a limited period of time. Memtrade does not require any modifications to host-level system software or support from the cloud provider. It harvests producer memory using an application-aware control loop to form a distributed transient remote memory pool with minimal performance impact; it employs a broker to match producers with consumers while satisfying performance constraints; and it exposes the matched memory to consumers as a secure KV cache. Our evaluation using real-world cluster traces shows that Memtrade provides significant performance benefit for consumers (improving average read latency up to 2.8x) while preserving confidentiality and integrity, with little impact on producer applications (degrading performance by less than 2.1%).
  }
}

@Article{tpp:arxiv22,
  author        = {Hasan Al Maruf and Hao Wang and Abhishek Dhanotia and Johannes Weiner and Niket Agarwal and Pallab Bhattacharya and Chris Petersen and Mosharaf Chowdhury and Shobhit Kanaujia and Prakash Chauhan},
  journal       = {CoRR},
  title         = {{TPP}: Transparent Page Placement for {CXL}-Enabled Tiered Memory},
  year          = {2022},
  month         = {Jun},
  volume        = {abs/2206.02878},
  archiveprefix = {arXiv},
  eprint        = {2206.02878},
  url           = {https://arxiv.org/abs/2206.02878},
  publist_confkey = {arXiv:2206.02878},
  publist_link = {paper || https://arxiv.org/abs/2206.02878},
  publist_topic = {Disaggregation},
  publist_abstract = {
    With increasing memory demands for datacenter applications and the emergence of coherent interfaces like CXL that enable main memory expansion, we are about to observe a wide adoption of tiered-memory subsystems in hyperscalers. In such systems, main memory can constitute different memory technologies with varied performance characteristics. In this paper, we characterize the memory usage of a wide range of datacenter applications across the server fleet of a hyperscaler (Meta) to get insights into an application's memory access patterns and performance on a tiered memory system. Our characterizations show that datacenter applications can benefit from tiered memory systems as there exist opportunities for offloading colder pages to slower memory tiers. Without efficient memory management, however, such systems can significantly degrade performance.
    We propose a novel OS-level application-transparent page placement mechanism (TPP) for efficient memory management. TPP employs a lightweight mechanism to identify and place hot and cold pages to appropriate memory tiers. It enables page allocation to work independently from page reclamation logic that is, otherwise, tightly coupled in today's Linux kernel. As a result, the local memory tier has memory headroom for new allocations. At the same time, TPP can promptly promote performance-critical hot pages trapped in the slow memory tiers to the fast tier node. Both promotion and demotion mechanisms work transparently without any prior knowledge of an application's memory access behavior. We evaluate TPP with diverse workloads that consume significant portions of DRAM on Meta's server fleet and are sensitive to memory subsystem performance. TPP's efficient page placement improves Linux's performance by up to 18%. TPP outperforms NUMA balancing and AutoTiering, state-of-the-art solutions for tiered memory, by 10-17%.
  }
}

@Article{egeria:arxiv22,
  author        = {Yiding Wang and Decang Sun and Kai Chen and Fan Lai and Mosharaf Chowdhury},
  journal       = {CoRR},
  title         = {Efficient {DNN} Training with Knowledge-Guided Layer Freezing},
  year          = {2022},
  month         = {Jan},
  volume        = {abs/2201.06227},
  archiveprefix = {arXiv},
  eprint        = {2201.06227},
  url           = {https://arxiv.org/abs/2201.06227},
  publist_confkey = {arXiv:2201.06227},
  publist_link = {paper || https://arxiv.org/abs/2201.06227},
  publist_topic = {Systems + AI},
  publist_abstract = {
    Training deep neural networks (DNNs) is time-consuming. While most existing solutions try to overlap/schedule computation and communication for efficient training, this paper goes one step further by skipping computing and communication through DNN layer freezing. Our key insight is that the training progress of internal DNN layers differs significantly, and front layers often become well-trained much earlier than deep layers. To explore this, we first introduce the notion of training plasticity to quantify the training progress of internal DNN layers. Then we design KGT, a knowledge-guided DNN training system that employs semantic knowledge from a reference model to accurately evaluate individual layers' training plasticity and safely freeze the converged ones, saving their corresponding backward computation and communication. Our reference model is generated on the fly using quantization techniques and runs forward operations asynchronously on available CPUs to minimize the overhead. In addition, KGT caches the intermediate outputs of the frozen layers with prefetching to further skip the forward computation. Our implementation and testbed experiments with popular vision and language models show that KGT achieves 19%-43% training speedup w.r.t. the state-of-the-art without sacrificing accuracy.
  }
}

@Article{treehouse-whitepaper:arxiv22,
  author        = {Thomas Anderson and Adam Belay and Mosharaf Chowdhury and Asaf Cidon and Irene Zhang},
  journal       = {CoRR},
  title         = {Treehouse: A Case For Carbon-Aware Datacenter Software},
  year          = {2022},
  month         = {Jan},
  volume        = {abs/2201.02120},
  archiveprefix = {arXiv},
  eprint        = {2201.02120},
  url           = {https://arxiv.org/abs/2201.02120},
  publist_confkey = {arXiv:2201.02120},
  publist_link = {paper || https://arxiv.org/abs/2201.02120},
  publist_topic = {Energy-Efficient Systems},
  publist_topic = {Disaggregation},
  publist_abstract = {
    The end of Dennard scaling and the slowing of Moore's Law has put the energy use of datacenters on an unsustainable path. Datacenters are already a significant fraction of worldwide electricity use, with application demand scaling at a rapid rate. We argue that substantial reductions in the carbon intensity of datacenter computing are possible with a software-centric approach: by making energy and carbon visible to application developers on a fine-grained basis, by modifying system APIs to make it possible to make informed trade offs between performance and carbon emissions, and by raising the level of application programming to allow for flexible use of more energy efficient means of compute and storage. We also lay out a research agenda for systems software to reduce the carbon footprint of datacenter computing.
  }
}

@Article{autops:arxiv22,
  author        = {Juncheng Gu and Mosharaf Chowdhury and Kang G. Shin and Aditya Akella},
  journal       = {CoRR},
  title         = {Elastic Model Aggregation with Parameter Service},
  year          = {2022},
  month         = {Apr},
  volume        = {abs/2204.03211},
  archiveprefix = {arXiv},
  eprint        = {2204.03211},
  url           = {https://arxiv.org/abs/2204.03211},
  publist_confkey = {arXiv:2204.03211},
  publist_link = {paper || https://arxiv.org/abs/2204.03211},
  publist_topic = {Systems + AI},
  publist_abstract = {
    Model aggregation, the process that updates model parameters, is an important step for model convergence in distributed deep learning (DDL). However, the parameter server (PS), a popular paradigm of performing model aggregation, causes CPU underutilization in deep learning (DL) clusters, due to the bursty nature of aggregation and static resource allocation. To remedy this problem, we propose Parameter Service, an elastic model aggregation framework for DDL training, which decouples the function of model aggregation from individual training jobs and provides a shared model aggregation service to all jobs in the cluster. In Parameter Service, model aggregations are efficiently packed and dynamically migrated to fit into the available CPUs with negligible time overhead. Furthermore, Parameter Service can elastically manage its CPU resources based on its load to enhance resource efficiency. We have implemented Parameter Service in a prototype system called AutoPS and evaluated it via testbed experimentation and trace-driven simulations. AutoPS reduces up to 75% of CPU consumption with little or no performance impact on the training jobs. The design of Parameter Service is transparent to the users and can be incorporated in popular DL frameworks.
  }
}

@Article{oort:arxiv20,
  author        = {Fan Lai and Xiangfeng Zhu and Harsha V. Madhyastha and Mosharaf Chowdhury},
  journal       = {CoRR},
  title         = {Oort: Informed Participant Selection for Scalable Federated Learning},
  year          = {2020},
  month         = {Oct},
  volume        = {abs/2010.06081},
  archiveprefix = {arXiv},
  bibsource     = {dblp computer science bibliography, https://dblp.org},
  biburl        = {https://dblp.org/rec/journals/corr/abs-2010-06081.bib},
  eprint        = {2010.06081},
  url           = {https://arxiv.org/abs/2010.06081},

  publist_confkey = {arXiv:2010.06081},
  publist_link = {paper || https://arxiv.org/abs/2010.06081},
  publist_topic = {Wide-Area Computing},
  publist_topic = {Systems + AI},
  publist_abstract = {
    Federated Learning (FL) is an emerging direction in distributed machine learning (ML) that enables in-situ model training and testing on edge data. Despite having the same end goals as traditional ML, FL executions differ significantly in scale, spanning thousands to millions of participating devices. As a result, data characteristics and device capabilities vary widely across clients. Yet, existing efforts randomly select FL participants, which leads to poor model and system efficiency.
    
    In this paper, we propose Kuiper to improve the performance of federated training and testing with guided participant selection. With an aim to improve time-to-accuracy performance in model training, Kuiper prioritizes the use of those clients who have both data that offers the greatest utility in improving model accuracy and the capability to run training quickly. To enable FL developers to interpret their results in model testing, Kuiper enforces their requirements on the distribution of participant data while improving the duration of federated testing by cherry-picking clients. Our evaluation shows that, compared to existing participant selection mechanisms, Kuiper improves time-to-accuracy performance by 1.2x-14.1x and final model accuracy by 1.3%-9.8%, while efficiently enforcing developer-specified model testing criteria at the scale of millions of clients.
  }
}

@Article{salus:arxiv19,
  author        = {Peifeng Yu and Mosharaf Chowdhury},
  journal       = {CoRR},
  title         = {Salus: Fine-Grained {GPU} Sharing Primitives for Deep Learning Applications},
  year          = {2019},
  month         = {Feb},
  volume        = {abs/1902.04610},
  archiveprefix = {arXiv},
  bibsource     = {dblp computer science bibliography, https://dblp.org},
  biburl        = {https://dblp.org/rec/journals/corr/abs-1902-04610.bib},
  eprint        = {1902.04610},
  url           = {https://arxiv.org/abs/1902.04610},

  publist_confkey = {arXiv:1902.04610},
  publist_link = {paper || https://arxiv.org/abs/1902.04610},
  publist_topic = {Systems + AI},
  publist_abstract = {
    GPU computing is becoming increasingly more popular with the proliferation of deep learning (DL) applications.
    However, unlike traditional resources such as CPU or the network, modern GPUs do not natively support fine-grained sharing primitives.
    Consequently, implementing common policies such as time sharing and preemption are expensive.
    Worse, when a DL application cannot completely use a GPU's resources, the GPU cannot be efficiently shared between multiple applications, leading to GPU underutilization.

    We present Salus to enable two GPU sharing primitives: fast job switching and memory sharing, in order to achieve fine-grained GPU sharing among multiple DL applications.
    Salus implements an efficient, consolidated execution service that exposes the GPU to different DL applications, and enforces fine-grained sharing by performing iteration scheduling and addressing associated memory management issues.
    We show that these primitives can then be used to implement flexible sharing policies such as fairness, prioritization, and packing for various use cases.
    Our integration of Salus with TensorFlow and evaluation on popular DL jobs show that Salus can improve the average completion time of DL training jobs by $3.19\times$,
    GPU utilization for hyper-parameter tuning by $2.38\times$,
    and GPU utilization of DL inference applications by $42\times$ over not sharing the GPU and $7\times$ over NVIDIA MPS with small overhead.
  }
}

@Article{terra:arxiv19,
  author        = {Jie You and Mosharaf Chowdhury},
  journal       = {CoRR},
  title         = {Terra: Scalable Cross-Layer {GDA} Optimizations},
  year          = {2019},
  month         = {Apr},
  volume        = {abs/1904.08480},
  archiveprefix = {arXiv},
  bibsource     = {dblp computer science bibliography, https://dblp.org},
  biburl        = {https://dblp.org/rec/journals/corr/abs-1904-08480.bib},
  eprint        = {1904.08480},
  url           = {https://arxiv.org/abs/1904.08480},
  publist_confkey = {arXiv:1904.08480},
  publist_link = {paper || https://arxiv.org/abs/1904.08480},
  publist_topic = {Wide-Area Computing},
  publist_abstract  = {Geo-distributed analytics (GDA) frameworks transfer large datasets over the wide-area network (WAN). Yet existing frameworks often ignore the WAN topology. This disconnect between WAN-bound applications and the WAN itself results in missed opportunities for cross-layer optimizations. In this paper, we present Terra to bridge this gap. Instead of decoupled WAN routing and GDA transfer scheduling, Terra applies scalable cross-layer optimizations to minimize WAN transfer times for GDA jobs. We present a two-pronged approach: (i) a scalable algorithm for joint routing and scheduling to make fast decisions; and (ii) a scalable, overlay-based enforcement mechanism that avoids expensive switch rule updates in the WAN. Together, they enable Terra to quickly react to WAN uncertainties such as large bandwidth fluctuations and failures in an application-aware manner as well. Integration with the FloodLight SDN controller and Apache YARN, and evaluation on 4 workloads and 3 WAN topologies show that Terra improves the average completion times of GDA jobs by 1.55x-3.43x. GDA jobs running with Terra meets 2.82x-4.29x more deadlines and can quickly react to WAN-level events in an application-aware manner.
  }
}

@Article{justitia:arxiv19,
  author        = {Yiwen Zhang and Yue Tan and Brent Stephens and Mosharaf Chowdhury},
  journal       = {CoRR},
  title         = {{RDMA} Performance Isolation With {Justitia}},
  year          = {2019},
  month         = {May},
  volume        = {abs/1905.04437},
  archiveprefix = {arXiv},
  bibsource     = {dblp computer science bibliography, https://dblp.org},
  biburl        = {https://dblp.org/rec/journals/corr/abs-1905-04437.bib},
  eprint        = {1905.04437},
  url           = {https://arxiv.org/abs/1905.04437},

  publist_confkey = {arXiv:1905.04437},
  publist_link = {paper || https://arxiv.org/abs/1905.04437},
  publist_topic = {Datacenter Networking},
  publist_topic = {Disaggregation},
  publist_abstract = {
    Despite its increasing popularity, most of RDMA’s benefits such as ultra-low latency can be achieved only when running an application in isolation. Using microbenchmarks and real open-source RDMA applications, we identify a series of performance anomalies when multiple applications coexist and show that such anomalies are pervasive across InfiniBand, RoCEv2, and iWARP. They arise due to a fundamental tradeoff between performance isolation and work conservation, which the state-of-the-art RDMA congestion control protocols such as DCQCN cannot resolve.

    We present Justitia to address these performance anomalies. Justitia is a software-only, host-based, and easy-to-deploy solution that maximizes RNIC utilization while guaranteeing performance isolation via shaping, rate limiting, and pacing at senders. Our evaluation of Justitia on multiple RDMA implementations show that Justitia effectively isolates different types of traffic and significantly improves latency (by up to 56.9×) and throughput (by up to 9.7×) of real-world RDMAbased applications without compromising low CPU usage or modifying the applications.
  }
}

@Article{nocs:arxiv19,
  author        = {Mosharaf Chowdhury and Samir Khuller and Manish Purohit and Sheng Yang and Jie You},
  journal       = {CoRR},
  title         = {Near Optimal Coflow Scheduling in Networks},
  year          = {2019},
  month         = {Jun},
  volume        = {abs/1906.06851},
  archiveprefix = {arXiv},
  bibsource     = {dblp computer science bibliography, https://dblp.org},
  biburl        = {https://dblp.org/rec/journals/corr/abs-1906-06851.bib},
  eprint        = {1906.06851},
  url           = {https://arxiv.org/abs/1906.06851},
  publist_confkey = {arXiv:1906.06851},
  publist_link = {paper || https://arxiv.org/abs/1906.06851},
  publist_topic = {Wide-Area Computing},
  publist_abstract = { The coflow scheduling problem has emerged as a popular abstraction in the last few years to study data communication problems within a data center. In this basic framework, each coflow has a set of communication demands and the goal is to schedule many coflows in a manner that minimizes the total weighted completion time. A coflow is said to complete when all its communication needs are met. This problem has been extremely well studied for the case of complete bipartite graphs that model a data center with full bisection bandwidth and several approximation algorithms and effective heuristics have been proposed recently.
In this work, we study a slightly different model of coflow scheduling in general graphs (to capture traffic between data centers) and develop practical and efficient approximation algorithms for it. Our main result is a randomized 2 approximation algorithm for the single path and free path model, significantly improving prior work. In addition, we demonstrate via extensive experiments that the algorithm is practical, easy to implement and performs well in practice.
}
}

@Article{hydra:arxiv19,
  author        = {Youngmoon Lee and Hasan Al Maruf and Mosharaf Chowdhury and Asaf Cidon and Kang G. Shin},
  journal       = {CoRR},
  title         = {Mitigating the Performance-Efficiency Tradeoff in Resilient Memory Disaggregation},
  year          = {2019},
  month         = {Oct},
  volume        = {abs/1910.09727},
  archiveprefix = {arXiv},
  bibsource     = {dblp computer science bibliography, https://dblp.org},
  biburl        = {https://dblp.org/rec/journals/corr/abs-1910-09727.bib},
  eprint        = {1910.09727},
  url           = {https://arxiv.org/abs/1910.09727},

  publist_confkey = {arXiv:1910.09727},
  publist_link = {paper || https://arxiv.org/abs/1910.09727},
  publist_topic = {Disaggregation},
  publist_abstract = {
    Memory disaggregation has received attention in recent years as a promising idea to reduce the total cost of ownership (TCO) of memory in modern datacenters. However, relying on remote memory expands an application’s failure domain and makes it susceptible to tail latency variations. In attempts to making disaggregated memory resilient, stateof-the-art solutions face the classic tradeoff between performance and efficiency: some double the memory overhead of disaggregation by replicating to remote memory, while many others limit performance by replicating to the local disk.

    We present Hydra, a configurable, erasure-coded resilience mechanism for common memory disaggregation solutions. It can transparently handle uncertainties arising from remote failures, evictions, memory corruptions, and stragglers from network imbalance with a significantly better performanceefficiency tradeoff than the state-of-the-art. We design a finetuned data path to achieve single µs read/write latency to remote memory, develop decentralized algorithms for clusterwide memory management, and analyze how to select parameters to mitigate independent and correlated uncertainties. Our integration of Hydra with two major memory disaggregation systems and evaluation on a 50-machine RDMA cluster demonstrates that it achieves the best of both worlds: it improves the latency and throughput of memory-intensive applications by up to 64.78× and 20.61×, respectively, over the state-of-the-art disk backup-based solution. At the same time, it provides performance similar to that of in-memory replication with 1.6× lower memory overhead.
  }
}

@Article{bopf:arxiv19,
  author        = {Tan N. Le and Xiao Sun and Mosharaf Chowdhury and Zhenhua Liu},
  journal       = {CoRR},
  title         = {{BoPF}: Mitigating the Burstiness-Fairness Tradeoff in Multi-Resource Clusters},
  year          = {2019},
  month         = {Dec},
  volume        = {abs/1912.03523},
  archiveprefix = {arXiv},
  bibsource     = {dblp computer science bibliography, https://dblp.org},
  biburl        = {https://dblp.org/rec/journals/corr/abs-1912-03523.bib},
  eprint        = {1912.03523},
  url           = {https://arxiv.org/abs/1912.03523},

  publist_confkey = {arXiv:1912.03523},
  publist_link = {paper || https://arxiv.org/abs/1912.03523},
  publist_topic = {Big Data Systems},
  publist_abstract = {
    Simultaneously supporting latency- and throughout-sensitive workloads in a shared environment is an increasingly more common challenge in big data clusters. Despite many advances, existing cluster schedulers force the same performance goal - fairness in most cases - on all jobs. Latency-sensitive jobs suffer, while throughput-sensitive ones thrive. Using prioritization does the opposite: it opens up a path for latency-sensitive jobs to dominate. In this paper, we tackle the challenges in supporting both short-term performance and long-term fairness simultaneously with high resource utilization by proposing Bounded Priority Fairness (BoPF). BoPF provides short-term resource guarantees to latency-sensitive jobs and maintains long-term fairness for throughput-sensitive jobs. BoPF is the first scheduler that can provide long-term fairness, burst guarantee, and Pareto efficiency in a strategyproof manner for multi-resource scheduling. Deployments and large-scale simulations show that BoPF closely approximates the performance of Strict Priority as well as the fairness characteristics of DRF. In deployments, BoPF speeds up latency-sensitive jobs by 5.38 times compared to DRF, while still maintaining long-term fairness. In the meantime, BoPF improves the average completion times of throughput-sensitive jobs by up to 3.05 times compared to Strict Priority.
  }
}

@InProceedings{allox:eurosys20,
  author    = {Tan N. Le and Xiao Sun and Mosharaf Chowdhury and Zhenhua Liu},
  booktitle = {ACM EuroSys},
  title     = {{AlloX}: Compute Allocation in Hybrid Clusters},
  year      = {2020},
  pages     = {31:1--31:16},

  publist_confkey = {EuroSys'20},
  publist_link = {paper || allox-eurosys20.pdf},
  publist_link = {code || https://github.com/lenhattan86/allox},
  publist_topic = {Systems + AI},
  publist_abstract = {
    Modern deep learning frameworks support a variety of hardware, including CPU, GPU, and other accelerators, to perform computation. In this paper, we study how to schedule jobs over such interchangeable resources – each with a different rate of computation – to optimize performance while providing fairness among users in a shared cluster. We demonstrate theoretically and empirically that existing solutions and their straightforward modifications perform poorly in the presence of interchangeable resources, which motivates the design and implementation of AlloX. At its core, AlloX transforms the scheduling problem into a min-cost bipartite matching problem and provides dynamic fair allocation over time. We theoretically prove its optimality in an ideal, offline setting and show empirically that it works well in the online scenario by incorporating with Kubernetes. Evaluations on a small-scale CPU-GPU hybrid cluster and large-scale simulations highlight that AlloX can reduce the average job completion time significantly (by up to 95% when the system load is high) while providing fairness and preventing starvation.  }
}

@InProceedings{salus:mlsys20,
  author    = {Peifeng Yu and Mosharaf Chowdhury},
  booktitle = {MLSys},
  title     = {Salus: Fine-Grained {GPU} Sharing Primitives for Deep Learning Applications},
  year      = {2020},

  publist_confkey = {MLSys'20},
  publist_link = {paper || salus-mlsys20.pdf},
  publist_link = {slides || salus-mlsys20-talk.pptm},
  publist_link = {poster || salus-mlsys20-poster.pdf},
  publist_badge = {Artifacts Available},
  publist_badge = {Artifacts Functional},
  publist_badge = {Results Reproduced},
  publist_topic = {Systems + AI},
  publist_abstract = {
    Unlike traditional resources such as CPU or the network, modern GPUs do not natively support
    fine-grained sharing primitives.
    Consequently, implementing common policies such as time sharing and preemption are expensive. Worse,
    when a deep learning (DL) application cannot completely use a GPU's resources, the GPU cannot be efficiently shared
    between multiple applications, leading to GPU underutilization.

    We present Salus to enable two GPU sharing primitives: __fast job
    switching__ and __memory sharing__, to achieve fine-grained GPU sharing
    among multiple DL applications. Salus is an efficient, consolidated
    execution service that exposes the GPU to different DL applications, and it
    enforces fine-grained sharing by performing iteration scheduling and
    addressing associated memory management issues. We show that these primitives
    can then be used to implement flexible sharing policies. Our integration of
    Salus with TensorFlow and evaluation on popular DL jobs shows that Salus
    can improve the average completion time of DL training jobs by $3.19\times$, GPU utilization for
    hyper-parameter tuning by $2.38\times$, and GPU utilization of DL inference applications by $42\times$ over not sharing
    the GPU and $6\times$ over NVIDIA MPS with small overhead.
  }
}

@InProceedings{sol:nsdi20,
  author    = {Fan Lai and Jie You and Xiangfeng Zhu and Harsha V. Madhyastha and Mosharaf Chowdhury},
  booktitle = {USENIX NSDI},
  title     = {Sol: Fast Distributed Computation Over Slow Networks},
  year      = {2020},
  pages     = {273--288},

  publist_confkey = {NSDI'20},
  publist_link = {paper || sol-nsdi20.pdf},
  publist_link = {slides || sol-nsdi20-slides.pdf},
  publist_link = {code || https://github.com/SymbioticLab/Sol},
  publist_topic = {Wide-Area Computing},
  publist_abstract = {
    The popularity of big data and AI has led to many optimizations at different layers of distributed computation stacks. Despite – or perhaps, because of – its role as the narrow waist of such software stacks, the design of the execution engine, which is in charge of executing every single task of a job, has mostly remained unchanged. As a result, the execution engines available today are ones primarily designed for low latency and high bandwidth datacenter networks. When either or both of the network assumptions do not hold, CPUs are significantly underutilized.

    In this paper, we take a first-principles approach toward developing an execution engine that can adapt to diverse network conditions. Sol, our federated execution engine architecture, flips the status quo in two respects. First, to mitigate the impact of high latency, Sol proactively assigns tasks, but does so judiciously to be resilient to uncertainties. Second, to improve the overall resource utilization, Sol decouples communication from computation internally instead of committing resources to both aspects of a task simultaneously. Our evaluations on EC2 show that, compared to Apache Spark in resource-constrained networks, Sol improves SQL and machine learning jobs by $16.4\times$ and $4.2\times$ on average.
  }
}

@InProceedings{pando:nsdi20,
  author    = {Muhammed Uluyol and Anthony Huang and Ayush Goel and Mosharaf Chowdhury and Harsha V. Madhyastha},
  booktitle = {USENIX NSDI},
  title     = {Near-Optimal Latency Versus Cost Tradeoffs in Geo-Distributed Storage},
  year      = {2020},
  pages     = {157--180},

  publist_confkey = {NSDI'20},
  publist_link = {paper || pando-nsdi20.pdf},
  publist_topic = {Wide-Area Computing},
  publist_abstract = {
    By replicating data across sites in multiple geo- graphic regions, web services can maximize availability and minimize latency for their users. However, when sacrificing data consistency is not an option, we show that service providers have to today incur significantly higher cost to meet desired la- tency goals than the lowest cost theoretically feasible. We show that the key to addressing this sub-optimality is to 1) allow for erasure coding, not just replication, of data across data cen- ters, and 2) mitigate the resultant increase in read and write la- tencies by rethinking how to enable consensus across the wide- area network. Our extensive evaluation mimicking web service deployments on the Azure cloud service shows that we enable near-optimal latency versus cost tradeoffs.
  }
}

@InProceedings{justitia:nsdi22,
  author    = {Yiwen Zhang and Yue Tan and Brent Stephens and Mosharaf Chowdhury},
  booktitle = {USENIX NSDI},
  title     = {{Justitia}: Software Multi-Tenancy in Hardware Kernel-Bypass Networks},
  year      = {2022},

  publist_confkey = {NSDI'22},
  publist_link = {paper || justitia-nsdi22.pdf},
  publist_link = {code || https://github.com/SymbioticLab/Justitia},
  publist_topic = {Datacenter Networking},
  publist_topic = {Disaggregation},
  publist_abstract = {
    Kernel-bypass networking (KBN) is becoming the new norm in modern datacenters.
    While hardware-based KBN offloads all dataplane tasks to specialized NICs to achieve better latency and CPU efficiency than software-based KBN, it also takes away the operator's control over network sharing policies.

    Providing policy support in multi-tenant hardware KBN brings unique challenges -- namely,
    preserving ultra-low latency and low CPU cost, finding a well-defined point of mediation, and rethinking traffic shapers.
    We present Justitia to address these challenges with three key design aspects:
    (i) Split Connection with message-level shaping,
    (ii) sender-based resource mediation together with receiver-side updates, and
    (iii) passive latency monitoring.
    Using a latency target as its knob, Justitia enables multi-tenancy policies such as predictable latencies and fair/weighted resource sharing.
    Our evaluation shows Justitia can effectively isolate latency-sensitive applications at the cost of slightly decreased utilization and ensure that throughput and bandwidth of the rest are not unfairly penalized.
  }
}

@InProceedings{aequitas:sigcomm22,
  author    = {Yiwen Zhang and Gautam Kumar and Nandita Dukkipati and Xian Wu and Priyaranjan Jha and Mosharaf Chowdhury and Amin Vahdat},
  booktitle = {ACM SIGCOMM},
  title     = {Aequitas: Admission Control for Performance-Critical {RPCs} in Datacenters},
  year      = {2022},

  publist_confkey = {SIGCOMM'22},
  publist_link = {paper || aequitas-sigcomm22.pdf},
  publist_link = {code || https://github.com/SymbioticLab/Aequitas},
  publist_topic = {Datacenter Networking},
  publist_topic = {Disaggregation},
  publist_badge = {Artifacts Available},
  publist_badge = {Artifacts Functional},
  publist_badge = {Results Reproduced},
  publist_abstract = {
    With the increasing popularity of disaggregated storage and microservice architectures, high fan-out and fan-in Remote Procedure Calls (RPCs) now generate most of the traffic in modern datacenters. 
    While the network plays a crucial role in RPC performance, traditional traffic classification categories cannot sufficiently capture their importance due to wide variations in RPC characteristics.
    As a result, meeting service-level objectives (SLOs), especially for performance-critical ($PC$) RPCs, remains challenging.

    We present Aequitas, a distributed sender-driven admission control scheme that uses commodity Weighted-Fair Queuing (WFQ) to guarantee RPC-level SLOs.
    In the presence of network overloads, it enforces cluster-wide RPC latency SLOs by limiting the amount of traffic admitted into any given QoS and downgrading the rest.
    We show analytically and empirically that this simple scheme works well.
    When the network demand spikes beyond provisioned capacity, Aequitas achieves a latency SLO that is 3.8$\times$ lower than the state-of-art congestion control at the 99.9$^{th}$-$p$ and admits up to 2$\times$ more $PC$ RPCs meeting SLO when compared with pFabric, Qjump, D$^3$, PDQ, and Homa.
    Results in our fleetwide production deployment show a 10\% latency improvement.
  }
}

@InProceedings{vulcan:nsdi24,
  author    = {Yiwen Zhang and Xumiao Zhang and Ganesh Ananthanarayanan and Anand Iyer and Yuanchao Shu and Victor Bahl and Z. Morley Mao and Mosharaf Chowdhury},
  booktitle = {USENIX NSDI},
  title     = {Vulcan: Automatic Query Planning for Live {ML} Analytics},
  year      = {2024},

  publist_confkey = {NSDI'24},
  publist_link = {paper || vulcan-nsdi24.pdf},
  publist_topic = {Systems + AI},
  publist_abstract = {
      Live ML analytics have gained increasing popularity with large-scale deployments due to recent evolution of ML technologies. To serve live ML queries, experts nowadays still need to perform manual query planning, which involves pipeline construction, query configuration, and pipeline placement across multiple edge tiers in a heterogeneous infrastructure. Finding the best query plan for a live ML query requires navigating a huge search space, calling for an efficient and systematic solution.

    In this paper, we propose Vulcan, a system that automatically generates query plans for live ML queries to optimize their accuracy, latency, and resource consumption. Based on the user query and performance requirements, Vulcan determines the best pipeline, placement, and query configuration for the query with low profiling cost; it also performs fast online adaptation after query deployment. Vulcan outperforms state-of-the-art ML analytics systems by 4.1$\times$-30.1$\times$ in terms of search cost while delivering up to 3.3$\times$ better query latency.
  }
}

@InProceedings{netlock:sigcomm20,
  author    = {Zhuolong Yu and Yiwen Zhang and Vladimir Braverman and Mosharaf Chowdhury and Xin Jin},
  booktitle = {ACM SIGCOMM},
  title     = {{NetLock}: Fast, Centralized Lock Management Using Programmable Switches},
  year      = {2020},
  pages     = {126--138},

  publist_confkey = {SIGCOMM'20},
  publist_link = {paper || netlock-sigcomm20.pdf},
  publist_link = {code || https://github.com/netx-repo/NetLock},
  publist_topic = {Datacenter Networking},
  publist_topic = {Disaggregation},
  publist_badge = {Artifacts Available},
  publist_badge = {Artifacts Functional},
  publist_badge = {Results Reproduced},
  publist_abstract = {
  
    Lock managers are widely used by distributed systems. Traditional centralized lock managers can easily support policies between multiple users using global knowledge, but they suffer from low performance. In contrast, emerging decentralized approaches are faster but cannot provide flexible policy support. Furthermore, performance in both cases is limited by the server capability.

    We present NetLock, a new centralized lock manager that co-designs servers and network switches to achieve high performance without sacrificing flexibility in policy support. The key idea of NetLock is to exploit the capability of emerging programmable switches to directly process lock requests in the switch data plane. Due to the limited switch memory, we design a memory management mechanism to seamlessly integrate the switch and server memory. To realize the locking functionality in the switch, we design a custom data plane module that efficiently pools multiple register arrays together to maximize memory utilization We have implemented a NetLock prototype with a Barefoot Tofino switch and a cluster of commodity servers. Evaluation results show that NetLock improves the throughput by 14.0-18.4x, and reduces the average and 99% latency by 4.7-20.3x and 10.4-18.7x over DSLR, a state-of-the-art RDMA-based solution, while providing flexible policy support.
  }
}

@InProceedings{leap:atc20,
  author    = {Hasan Al Maruf and Mosharaf Chowdhury},
  booktitle = {USENIX ATC},
  title     = {Effectively Prefetching Remote Memory with {Leap}},
  year      = {2020},
  pages     = {843--857},

  publist_confkey = {ATC'20},
  publist_link = {paper || leap-atc20.pdf},
  publist_link = {slides || leap-atc20-slides.pdf},
  publist_link = {code || https://github.com/SymbioticLab/Leap},
  publist_badge = {Best Paper Award},
  publist_topic = {Disaggregation},
  publist_abstract = {
    Memory disaggregation over RDMA can improve the performance of memory-constrained applications by replacing disk swapping with remote memory accesses. However, state-of-the-art memory disaggregation solutions still use data path components designed for slow disks. As a result, applications experience remote memory access latency significantly higher than that of the underlying low-latency network, which itself can be too high for many applications.

    In this paper, we propose Leap, a prefetching solution for remote memory accesses due to memory disaggregation. At its core, Leap employs an online, majority-based prefetching algorithm, which increases the page cache hit rate. We complement it with a lightweight and efficient data path in the kernel that isolates each application’s data path to the disaggregated memory and mitigates latency bottlenecks arising from legacy throughput-optimizing operations. Integration of Leap in the Linux kernel improves the median and tail remote page access latencies of memory-bound applications by up to 104.04× and 22.62×, respectively, over the default data path. This leads to up to 10.16× performance improvements for applications using disaggregated memory in comparison to the state-of-the-art solutions.
  }
}

@InProceedings{hydra:fast22,
  author    = {Youngmoon Lee and Hasan Al Maruf and Mosharaf Chowdhury and Asaf Cidon and Kang G. Shin},
  booktitle = {USENIX FAST},
  title     = {{Hydra} : Resilient and Highly Available Remote Memory},
  year      = {2022},

  publist_confkey = {FAST'22},
  publist_link = {paper || hydra-fast22.pdf},
  publist_link = {code || https://github.com/SymbioticLab/hydra},
  publist_topic = {Disaggregation},
  publist_abstract = {
    We present Hydra, a low-latency, low-overhead, and highly available resilience mechanism for remote memory. Hydra can access erasure-coded remote memory within a single-digit $\mu$s read/write latency, significantly improving the performance-efficiency tradeoff over the state-of-the-art -- it performs similar to in-memory replication with 1.6$\times$ lower memory overhead. We also propose CodingSets, a novel coding group placement algorithm for erasure-coded data, that provides load balancing while reducing the probability of data loss under correlated failures by an order of magnitude. With Hydra, even when only 50\% memory is local, unmodified memory-intensive applications achieve performance close to that of the fully in-memory case in the presence of remote failures and outperforms the state-of-the-art remote-memory solutions by up to 4.35$\times$.
  }
}

@InProceedings{tpp:asplos23,
  author    = {Hasan Al Maruf and Hao Wang and Abhishek Dhanotia and Johannes Weiner and Niket Agarwal and Pallab Bhattacharya and Chris Petersen and Mosharaf Chowdhury and Shobhit Kanaujia and Prakash Chauhan},
  booktitle = {ASPLOS},
  title     = {{TPP}: Transparent Page Placement for {CXL}-Enabled Tiered-Memory},
  year      = {2023},

  publist_confkey = {ASPLOS'23},
  publist_badge = {MICRO Top Picks Honorable Mention},
  publist_link = {paper || tpp-asplos23.pdf},
  publist_link = {slides || tpp-slides-asplos23.pdf},
  publist_link = {code || https://lwn.net/Articles/876993/},
  publist_topic = {Disaggregation},
  publist_abstract = {
    The increasing demand for memory in hyperscale applications has led to memory becoming a large portion of the overall datacenter spend. The emergence of coherent interfaces like CXL enables main memory expansion and offers an efficient solution to this problem. In such systems, the main memory can constitute different memory technologies with varied characteristics. In this paper, we characterize memory usage patterns of a wide range of datacenter applications across the server fleet of Meta. We, therefore, demonstrate the opportunities to offload colder pages to slower memory tiers for these applications. Without efficient memory management, however, such systems can significantly degrade performance.

We propose a novel OS-level application-transparent page placement mechanism (TPP) for CXL-enabled memory. TPP employs a lightweight mechanism to identify and place hot/cold pages to appropriate memory tiers. It enables a proactive page demotion from local memory to CXL-Memory. This technique ensures a memory headroom for new page allocations that are often related to request processing and tend to be short-lived and hot. At the same time, TPP can promptly promote performance-critical hot pages trapped in the slow CXL-Memory to the fast local memory, while minimizing both sampling overhead and unnecessary migrations. TPP works transparently without any application-specific knowledge and can be deployed globally as a kernel release.

We evaluate TPP with diverse memory-sensitive workloads in the production server fleet with early samples of new x86 CPUs with CXL 1.1 support. TPP makes a tiered memory system performant as an ideal baseline (<1% gap) that has all the memory in the local tier. It is 18% better than today’s Linux, and 5–17% better than existing solutions including NUMA Balancing and AutoTiering. Most of the TPP patches have been merged in the Linux v5.18 release while the remaining ones are just pending for more discussion.
  }
}

@InProceedings{memtrade:sigmetrics23,
  author        = {Hasan Al Maruf and Yuhong Zhong and Hongyi Wang and Mosharaf Chowdhury and Asaf Cidon and Carl Waldspurger},
  booktitle 	= {SIGMETRICS},
  title         = {Memtrade: Marketplace for Disaggregated Memory Clouds},
  year          = {2023},
  publist_confkey = {SIGMETRICS'23},
  publist_link = {paper || memtrade-sigmetrics23.pdf},
  publist_link = {code || https://github.com/SymbioticLab/Memtrade},
  publist_topic = {Disaggregation},
  publist_abstract = {
    We present Memtrade, the first practical marketplace for disaggregated memory clouds. Clouds introduce a set of unique challenges for resource disaggregation across different tenants, including resource harvesting, isolation, and matching. Memtrade allows producer virtual machines (VMs) to lease both their unallocated memory and allocated-but-idle application memory to remote consumer VMs for a limited period of time. Memtrade does not require any modifications to host-level system software or support from the cloud provider. It harvests producer memory using an application-aware control loop to form a distributed transient remote memory pool with minimal performance impact; it employs a broker to match producers with consumers while satisfying performance constraints; and it exposes the matched memory to consumers through different abstractions. As a proof of concept, we propose two such memory access interfaces for Memtrade consumers -- a transient KV cache for specified applications and a swap interface that is application-transparent. Our evaluation using real-world cluster traces shows that Memtrade provides significant performance benefit for consumers (improving average read latency up to 2.8X) while preserving confidentiality and integrity, with little impact on producer applications (degrading performance by less than 2.1%).
  }
}

@article{hsct-survey,
  title={A Systematic Review of Machine Learning Techniques in Hematopoietic Stem Cell Transplantation ({HSCT})},
  author={Gupta, Vibhuti and Braun, Thomas M and Chowdhury, Mosharaf and Tewari, Muneesh and Choi, Sung Won},
  journal={Sensors},
  volume={20},
  number={21},
  pages={6100},
  year={2020},
  publisher={Multidisciplinary Digital Publishing Institute},
  publist_confkey = {Sensors:20(21)},
  publist_link = {paper || https://www.mdpi.com/1424-8220/20/21/6100},
  publist_abstract = {
    Machine learning techniques are widely used nowadays in the healthcare domain for the diagnosis, prognosis, and treatment of diseases. These techniques have applications in the field of hematopoietic cell transplantation (HCT), which is a potentially curative therapy for hematological malignancies. Herein, a systematic review of the application of machine learning (ML) techniques in the HCT setting was conducted. We examined the type of data streams included, specific ML techniques used, and type of clinical outcomes measured. A systematic review of English articles using PubMed, Scopus, Web of Science, and IEEE Xplore databases was performed. Search terms included "hematopoietic cell transplantation (HCT)," "autologous HCT," "allogeneic HCT," "machine learning," and "artificial intelligence." Only full-text studies reported between January 2015 and July 2020 were included. Data were extracted by two authors using predefined data fields. Following PRISMA guidelines, a total of 242 studies were identified, of which 27 studies met the inclusion criteria. These studies were sub-categorized into three broad topics and the type of ML techniques used included ensemble learning (63%), regression (44%), Bayesian learning (30%), and support vector machine (30%). The majority of studies examined models to predict HCT outcomes (e.g., survival, relapse, graft-versus-host disease). Clinical and genetic data were the most commonly used predictors in the modeling process. Overall, this review provided a systematic review of ML techniques applied in the context of HCT. The evidence is not sufficiently robust to determine the optimal ML technique to use in the HCT setting and/or what minimal data variables are required.  
  }
}

@InProceedings{kayak:nsdi21,
  author    = {Jie You and Jingfeng Wu and Xin Jin and Mosharaf Chowdhury},
  booktitle = {USENIX NSDI},
  title     = {Ship Compute or Ship Data? Why Not Both?},
  year      = {2021},

  publist_confkey = {NSDI'21},
  publist_link = {paper || kayak-nsdi21.pdf},
  publist_link = {code || https://github.com/SymbioticLab/Kayak},
  publist_topic = {Disaggregation},
  publist_abstract = {
    How cloud applications should interact with their data re-mains an active area of research. Over the last decade, manyhave suggested relying on a key-value (KV) interface to inter-act with data stored in remote storage servers, while othershave vouched for the benefits of using remote procedure call(RPC). Instead of choosing one over the other, in this paper,we observe that an ideal solution must adaptively combineboth of them in order to maximize throughput while meetingapplication latency requirements. To this end, we proposea new system called Kayak that proactively adjusts the rateof requests and the fraction of requests to be executed usingRPC or KV, all in a fully decentralized and self-regulated man-ner. We theoretically prove that Kayak can quickly convergeto the optimal parameters.  We implement a system proto-type of Kayak. Our evaluations show that Kayak achievessub-second convergence  and improves  overall throughputby 32.5%-63.4% for compute-intensive workloads and upto 12.2% for non-compute-intensive and transactional work-loads over the state-of-the-art.
  }
}

@article{fedscale:arxiv21,
  title={FedScale: Benchmarking Model and System Performance of Federated Learning},
  author={Lai, Fan and Dai, Yinwei and Zhu, Xiangfeng and Chowdhury, Mosharaf},
  archiveprefix = {arXiv},
  eprint= {2105.11367},
  url= {https://arxiv.org/abs/2105.11367},
  year={2021},
  month={May},
  publist_confkey = {arXiv:2105.11367},
  publist_link = {paper || https://arxiv.org/abs/2105.11367},
  publist_link = {code || https://github.com/SymbioticLab/FedScale},
  publist_link = {website || https://fedscale.ai/},
  publist_topic = {Wide-Area Computing},
  publist_topic = {Systems + AI},
  publist_abstract = {
    We present FedScale, a diverse set of challenging and realistic benchmark datasets to facilitate scalable, comprehensive, and reproducible 
    federated learning (FL) research. FedScale datasets are large-scale, encompassing a diverse range of important FL tasks, such as image 
    classification, object detection, language modeling, speech recognition, and reinforcement learning. For each dataset, we provide a unified
    evaluation protocol using realistic data splits and evaluation metrics. To meet the pressing need for reproducing realistic FL at scale,
    
    we have also built an efficient evaluation platform to simplify and standardize the process of FL experimental setup and model evaluation.
    Our evaluation platform provides flexible APIs to implement new FL algorithms and include new execution backends with minimal developer 
    efforts. Finally, we perform indepth benchmark experiments on these datasets. Our experiments suggest that FedScale presents significant
    challenges of heterogeneity-aware co-optimizations of the system and statistical efficiency under realistic FL characteristics, 
    indicating fruitful opportunities for future research. FedScale is open-source with permissive licenses and actively maintained, and 
    we welcome feedback and contributions from the community.
  }
}

@article{fed-ensemble:arxiv21,
  title={Fed-ensemble: Improving Generalization through Model Ensembling in Federated Learning}, 
  author={Naichen Shi and Fan Lai and Raed Al Kontar and Mosharaf Chowdhury},
  archiveprefix = {arXiv},
  eprint= {2107.10663},
  url= {https://arxiv.org/abs/2107.10663},
  year={2021},
  month={July},
  publist_confkey = {arXiv:2107.10663},
  publist_link = {paper || https://arxiv.org/abs/2107.10663},
  publist_topic = {Wide-Area Computing},
  publist_abstract = {
    In this paper we propose Fed-ensemble: a simple approach that brings model ensembling to
    federated learning (FL). Instead of aggregating local models to update a single global model, Fed-ensemble 
    uses random permutations to update a group of K models and then obtains predictions
    through model averaging. Fed-ensemble can be readily utilized within established FL methods and
    does not impose a computational overhead as it only requires one of the K models to be sent to a
    client in each communication round. Theoretically, we show that predictions on new data from all K
    models belong to the same predictive posterior distribution under a neural tangent kernel regime.
    This result in turn sheds light on the generalization advantages of model averaging. We also illustrate
    that Fed-ensemble has an elegant Bayesian interpretation. Empirical results show that our model
    has superior performance over several FL algorithms, on a wide range of data sets, and excels in
    heterogeneous settings often encountered in FL applications.
  }
}

@article{ioft-survey:arxiv21,
  title={{The Internet of Federated Things (IoFT)}: A Vision for the Future and In-Depth Survey of Data-Driven Approaches for Federated Learning}, 
  author={Raed Kontar and Naichen Shi and Xubo Yue and Seokhyun Chung and Eunshin Byon and Mosharaf Chowdhury and Judy Jin and Wissam Kontar and Neda Masoud and Maher Noueihed and Chinedum E Okwudire and Garvesh Raskutti and Romesh Saigal and Karandeep Singh and Zhisheng Ye},
  archiveprefix = {arXiv},
  eprint= {2111.05326},
  url= {https://arxiv.org/abs/2111.05326},
  year={2021},
  month={November},
  publist_confkey = {arXiv:2111.05326},
  publist_link = {paper || https://arxiv.org/abs/2111.05326},
  publist_topic = {Wide-Area Computing},
  publist_abstract = {
    The Internet of Things (IoT) is on the verge of a major paradigm shift. In the IoT system of the future, IoFT, the cloud will be substituted by the crowd where model training is brought to the edge, allowing IoT devices to collaboratively extract knowledge and build smart analytics/models while keeping their personal data stored locally. This paradigm shift was set into motion by the tremendous increase in computational power on IoT devices and the recent advances in decentralized and privacy-preserving model training, coined as federated learning (FL). This article provides a vision for IoFT and a systematic overview of current efforts towards realizing this vision. Specifically, we first introduce the defining characteristics of IoFT and discuss FL data-driven approaches, opportunities, and challenges that allow decentralized inference within three dimensions: (i) a global model that maximizes utility across all IoT devices, (ii) a personalized model that borrows strengths across all devices yet retains its own model, (iii) a meta-learning model that quickly adapts to new devices or learning tasks. We end by describing the vision and challenges of IoFT in reshaping different industries through the lens of domain experts. Those industries include manufacturing, transportation, energy, healthcare, quality & reliability, business, and computing.
  }
}

@article{ioft-survey:ieee-access,
  title={The {I}nternet of {F}ederated {T}hings ({IoFT})}, 
  author={Raed Kontar and Naichen Shi and Xubo Yue and Seokhyun Chung and Eunshin Byon and Mosharaf Chowdhury and Judy Jin and Wissam Kontar and Neda Masoud and Maher Noueihed and Chinedum E Okwudire and Garvesh Raskutti and Romesh Saigal and Karandeep Singh and Zhisheng Ye},
  journal={IEEE Access},
  volume={9},
  pages={156071--156113},
  year={2021},
  publisher={IEEE},
  publist_confkey = {IEEE Access:9},
  publist_link = {paper || https://doi.org/10.1109/ACCESS.2021.3127448},
  publist_badge = {Featured Article},
  publist_topic = {Wide-Area Computing},
  publist_abstract = {
    The Internet of Things (IoT) is on the verge of a major paradigm shift. In the IoT system of the future, IoFT, the cloud will be substituted by the crowd where model training is brought to the edge, allowing IoT devices to collaboratively extract knowledge and build smart analytics/models while keeping their personal data stored locally. This paradigm shift was set into motion by the tremendous increase in computational power on IoT devices and the recent advances in decentralized and privacy-preserving model training, coined as federated learning (FL). This article provides a vision for IoFT and a systematic overview of current efforts towards realizing this vision. Specifically, we first introduce the defining characteristics of IoFT and discuss FL data-driven approaches, opportunities, and challenges that allow decentralized inference within three dimensions: (i) a global model that maximizes utility across all IoT devices, (ii) a personalized model that borrows strengths across all devices yet retains its own model, (iii) a meta-learning model that quickly adapts to new devices or learning tasks. We end by describing the vision and challenges of IoFT in reshaping different industries through the lens of domain experts. Those industries include manufacturing, transportation, energy, healthcare, quality & reliability, business, and computing.
  }
}

@Article{covid-f20:jmirmh,
    author={Gilley, Kristen N and Baroudi, Loubna and Yu, Miao and Gainsburg, Izzy and Reddy, Niyanth and Bradley, Christina and Cislo, Christine and Rozwadowski, Michelle Lois and Clingan, Caroline Ashley and DeMoss, Matthew Stephen and Churay, Tracey and Birditt, Kira and Colabianchi, Natalie and Chowdhury, Mosharaf and Forger, Daniel and Gagnier, Joel and Zernicke, Ronald F and Cunningham, Julia Lee and Cain, Stephen M and Tewari, Muneesh and Choi, Sung Won},
    title={Risk Factors for {COVID-19} in College Students Identified by Physical, Mental, and Social Health Reported During the {Fall 2020} Semester: Observational Study Using the {Roadmap} App and {Fitbit} Wearable Sensors},
    journal={JMIR Mental Health},
    year={2022},
    month={Feb},
    day={10},
    volume={9},
    number={2},
    pages={e34645},
    publisher={JMIR Publications},
    publist_confkey = {JMIR-MH:9(2)},
    publist_link = {paper || https://doi.org/10.2196/34645},
    abstract={Background: The COVID-19 pandemic triggered a seismic shift in education to web-based learning. With nearly 20 million students enrolled in colleges across the United States, the long-simmering mental health crisis in college students was likely further exacerbated by the pandemic. Objective: This study leveraged mobile health (mHealth) technology and sought to (1) characterize self-reported outcomes of physical, mental, and social health by COVID-19 status; (2) assess physical activity through consumer-grade wearable sensors (Fitbit); and (3) identify risk factors associated with COVID-19 positivity in a population of college students prior to release of the vaccine. Methods: After completing a baseline assessment (ie, at Time 0 [T0]) of demographics, mental, and social health constructs through the Roadmap 2.0 app, participants were instructed to use the app freely, wear the Fitbit, and complete subsequent assessments at T1, T2, and T3, followed by a COVID-19 assessment of history and timing of COVID-19 testing and diagnosis (T4: {\textasciitilde}14 days after T3). Continuous measures were described using mean (SD) values, while categorical measures were summarized as n ({\%}) values. Formal comparisons were made on the basis of COVID-19 status. The multivariate model was determined by entering all statistically significant variables (P<.05) in univariable associations at once and then removing one variable at a time through backward selection until the optimal model was obtained. Results: During the fall 2020 semester, 1997 participants consented, enrolled, and met criteria for data analyses. There was a high prevalence of anxiety, as assessed by the State Trait Anxiety Index, with moderate and severe levels in 465 (24{\%}) and 970 (49{\%}) students, respectively. Approximately one-third of students reported having a mental health disorder (n=656, 33{\%}). The average daily steps recorded in this student population was approximately 6500 (mean 6474, SD 3371). Neither reported mental health nor step count were significant based on COVID-19 status (P=.52). Our analyses revealed significant associations of COVID-19 positivity with the use of marijuana and alcohol (P=.02 and P=.046, respectively) and with lower belief in public health measures (P=.003). In addition, graduate students were less likely and those with ≥20 roommates were more likely to report a COVID-19 diagnosis (P=.009). Conclusions: Mental health problems were common in this student population. Several factors, including substance use, were associated with the risk of COVID-19. These data highlight important areas for further attention, such as prioritizing innovative strategies that address health and well-being, considering the potential long-term effects of COVID-19 on college students. Trial Registration: ClinicalTrials.gov NCT04766788; https://clinicaltrials.gov/ct2/show/NCT04766788 International Registered Report Identifier (IRRID): RR2-10.2196/29561 },
    issn={2368-7959},
    doi={10.2196/34645}
}


@InProceedings{fluid:mlsys21,
  author    = {Peifeng Yu and Jiachen Liu and Mosharaf Chowdhury},
  booktitle = {MLSys},
  title     = {Fluid: Resource-Aware Hyperparameter Tuning Engine},
  year      = {2021},

  publist_confkey = {MLSys'21},
  publist_link = {paper || fluid-mlsys21.pdf},
  publist_link = {slides || fluid-mlsys21.pptx},
  publist_link = {poster || fluid-mlsys21-poster.pdf},
  publist_link = {code || https://github.com/SymbioticLab/fluid},
  publist_link = {video (oral) || https://www.youtube.com/watch?v=LZQE90whvfY},
  publist_link = {video (poster) || https://www.youtube.com/watch?v=yaBxEUxRXZI},
  publist_topic = {Systems + AI},
  publist_abstract = {
    Current hyperparameter tuning solutions lack complementary execution engines to efficiently leverage distributed
    computation, thus ignoring the possibility of intra- and inter-GPU sharing, which exhibits poor resource usage.
    In this paper, we present Fluid, a generalized hyperparameter tuning execution engine, that coordinates between
    hyperparameter tuning jobs and cluster resources. Fluid schedules evaluation trials in such jobs using a waterfilling
    approach to make the best use of resources both at intra- and inter-GPU granularities to speed up the tuning
    process. By abstracting a hyperparameter tuning job as a sequence of TrialGroup, Fluid can boost the performance
    of diverse hyperparameter tuning solutions. Our experiments show that Fluid can speed up synchronous BOHB by
    100\%, and BOHB and ASHA by 30\% while having similar final accuracy.
  }
}

@InProceedings{oort:osdi21,
  author    = {Fan Lai and Xiangfeng Zhu and Harsha V. Madhyastha and Mosharaf Chowdhury},
  booktitle = {USENIX OSDI},
  title     = {Oort: Efficient Federated Learning via Guided Participant Selection},
  year      = {2021},

  publist_confkey = {OSDI'21},
  publist_link = {paper || oort-osdi21.pdf},
  publist_link = {code || https://github.com/SymbioticLab/Oort},
  publist_badge = {Artifacts Available},
  publist_badge = {Artifacts Functional},
  publist_badge = {Results Reproduced},
  publist_badge = {Distinguished Artifact Award},
  publist_topic = {Wide-Area Computing},
  publist_topic = {Systems + AI},
  publist_abstract = {
    Federated Learning (FL) is an emerging direction in distributed machine learning (ML) that enables in-situ model training and testing on edge data. Despite having the same end goals as traditional ML, FL executions differ significantly in scale, spanning thousands to millions of participating devices. As a result, data characteristics and device capabilities vary widely across clients. Yet, existing efforts randomly select FL participants, which leads to poor model and system efficiency.
    In this paper, we propose Kuiper to improve the performance of federated training and testing with guided participant selection. With an aim to improve time-to-accuracy performance in model training, Kuiper prioritizes the use of those clients who have both data that offers the greatest utility in improving model accuracy and the capability to run training quickly. To enable FL developers to interpret their results in model testing, Kuiper enforces their requirements on the distribution of participant data while improving the duration of federated testing by cherry-picking clients. Our evaluation shows that, compared to existing participant selection mechanisms, Kuiper improves time-to-accuracy performance by 1.2×-14.1× and final model accuracy by 1.3%-9.8%, while efficiently enforcing developer-specified model testing criteria at the scale of millions of clients.
  }
}

@InProceedings{fedscale:resilientfl21,
  author    = {Fan Lai and Yinwei Dai and Xiangfeng Zhu and Harsha V. Madhyastha and Mosharaf Chowdhury},
  booktitle = {ACM SOSP ResilientFL},
  title     = {FedScale: Benchmarking Model and System Performance of Federated Learning},
  year      = {2021},

  publist_confkey = {ResilientFL'21},
  publist_badge = {Best Paper Award},
  publist_link = {paper || fedscale-resilientfl21.pdf},
  publist_link = {code || https://github.com/SymbioticLab/fedscale},
  publist_link = {website || https://fedscale.ai/},
  publist_topic = {Wide-Area Computing},
  publist_topic = {Systems + AI},
}

@InProceedings{treehouse:hotcarbon22,
  author    = {Thomas Anderson and Adam Belay and Mosharaf Chowdhury and Asaf Cidon and Irene Zhang},
  booktitle = {HotCarbon},
  title     = {Treehouse: A Case For Carbon-Aware Datacenter Software},
  year      = {2022},

  publist_confkey = {HotCarbon'22},
  publist_link = {paper || treehouse-hotcarbon22.pdf},
  publist_topic = {Energy-Efficient Systems},
  publist_abstract = {The end of Dennard scaling and the slowing of Moore's Law has put the energy use of datacenters on an unsustainable path. Datacenters are already a significant fraction of worldwide electricity use, with application demand scaling at a rapid rate. We argue that substantial reductions in the carbon intensity of datacenter computing are possible with a software-centric approach: by making energy and carbon visible to application developers on a fine-grained basis, by modifying system APIs to make it possible to make informed trade offs between performance and carbon emissions, and by raising the level of application programming to allow for flexible use of more energy efficient means of compute and storage. We also lay out a research agenda for systems software to reduce the carbon footprint of datacenter computing.
  }
}

@InProceedings{aifo:sigcomm21,
  author    = {Zhuolong Yu and Chuheng Hu and Jingfeng Wu and Xiao Sun and Vladimir Braverman and Mosharaf Chowdhury and Zhenhua Liu and Xin Jin},
  booktitle = {ACM SIGCOMM},
  title     = {Programmable Packet Scheduling with a Single Queue},
  year      = {2021},

  publist_confkey = {SIGCOMM'21},
  publist_link = {paper || aifo-sigcomm21.pdf},
  publist_link = {code || https://github.com/netx-repo/AIFO},
  publist_topic = {Datacenter Networking},
  publist_badge = {Artifacts Available},
  publist_badge = {Artifacts Functional},
  publist_badge = {Results Reproduced},
  publist_abstract = {
  
    Programmable packet scheduling enables scheduling algorithms to be programmed into the data plane without changing the hardware. Existing proposals either have no hardware implementations for switch ASICs or require multiple strict-priority queues.

    We present Admission-In First-Out (AIFO) queues, a new solution for programmable packet scheduling that uses only a single first-in first-out queue. AIFO is motivated by the confluence of two recent trends: shallow buffers in switches and fast-converging congestion control in end hosts, that together leads to a simple observation: the decisive factor in a flow’s completion time (FCT) in modern datacenter networks is often which packets are enqueued or dropped, not the ordering they leave the switch. The core idea of AIFO is to maintain a sliding window to track the ranks of recent packets and compute the relative rank of an arriving packet in the window for admission control. Theoretically, we prove that AIFO provides bounded performance to Push-In First-Out (PIFO). Empirically, we fully implement AIFO and evaluate AIFO with a range of real workloads, demonstrating AIFO closely approximates PIFO. Importantly, unlike PIFO, AIFO can run at line rate on existing hardware and use minimal switch resources---as few as a single queue.
  }
}

@inproceedings{fedscale:icml22,
  author={Fan Lai and Yinwei Dai and Sanjay S. Singapuram and Jiachen Liu and Xiangfeng Zhu and Harsha V. Madhyastha and Mosharaf Chowdhury},
  title={{FedScale}: Benchmarking Model and System Performance of Federated Learning at Scale},
  booktitle={ICML},
  year={2022},
  
  publist_confkey = {ICML'22},
  publist_link = {paper || fedscale-icml22.pdf},
  publist_link = {code || https://github.com/SymbioticLab/fedscale},
  publist_link = {website || https://fedscale.ai/},
  publist_topic = {Wide-Area Computing},
  publist_topic = {Systems + AI},
  publist_abstract = {
    We present FedScale, a federated learning (FL) benchmarking suite with realistic datasets and a scalable runtime to enable reproducible FL research. FedScale datasets encompass a wide
    range of critical FL tasks, ranging from image classification and object detection to language modeling and speech recognition. Each dataset comes with a unified evaluation protocol using
    real-world data splits and evaluation metrics. To reproduce realistic FL behavior, FedScale contains a scalable and extensible runtime. 
    It provides high-level APIs to implement FL algorithms, deploy them at scale across diverse hardware and software backends, and evaluate them at scale, all with minimal developer efforts. 
    We combine the two to perform systematic benchmarking experiments and highlight potential opportunities for heterogeneity-aware co-optimizations in FL. 
    FedScale is open-source and actively maintained by contributors from different institutions at http://fedscale.ai. We welcome feedback and contributions from the community.
  }
}


@PhDThesis{jcgu:dissertation,
  Title                    = {Efficient Resource Management for Deep Learning Clusters},
  Author                   = {Juncheng Gu},
  Institution              = {University of Michigan},
  Year                     = {2021},
  Month                    = {August},

  publist_confkey = {dissertation},
  publist_link = {paper || jcgu-dissertation.pdf},
  Abstract                 = {Deep Learning (DL) is gaining rapid popularity in various domains, such as computer vision, speech recognition, etc. With the increasing demands, large clusters have been built to develop DL models (i.e., data preparation and model training). DL jobs have some unique features ranging from their hardware requirements to execution patterns. However, the resource management techniques applied in existing DL clusters have not yet been adapted to those new features, which leads to resource inefficiency and hurts the perfor- mance of DL jobs.
  We observed three major challenges brought by DL jobs. First, data preparation jobs, which prepare training datasets from a large volume of raw data, are memory intensive. DL clusters often over-allocate memory resource to those jobs for protecting their performance, which causes memory underutilization in DL clusters. Second, the execution time of a DL training job is often unknown before job completion. Without such information, existing cluster schedulers are unable to minimize the average Job Completion Time (JCT) of those jobs. Third, model aggregations in Distributed Deep Learning (DDL) training are often assigned with a fixed group of CPUs. However, a large portion of those CPUs are wasted because the bursty model aggregations can not saturate them all the time.
  In this thesis, we propose a suite of techniques to eliminate the mismatches between DL jobs and resource management in DL clusters. First, we bring the idea of memory disaggregation to enhance the memory utilization of DL clusters. The unused memory in data preparation jobs is exposed as remote memory to other machines that are running out of local memory. Second, we design a two-dimensional attained-service-based scheduler to optimize the average JCT of DL training jobs. This scheduler takes the temporal and spatial characteristics of DL training jobs into consideration and can efficiently schedule them without knowing their execution time. Third, we define a shared model aggregation service to reduce the CPU cost of DDL training. Using this service, model aggregations from different DDL training jobs are carefully packed together and use the same group of CPUs in a time-sharing manner. With these techniques, we demonstrate that huge improvements in resource efficiency and job performance can be obtained when the cluster’s resource management matches with the features of DL jobs.}
}

@PhDThesis{peifeng:dissertation,
  Title                    = {Application-Aware Scheduling in Deep Learning Software Stacks},
  Author                   = {Peifeng Yu},
  Institution              = {University of Michigan},
  Year                     = {2022},
  Month                    = {August},

  publist_confkey          = {dissertation},
  publist_link             = {paper || peifeng-dissertation.pdf},
  publist_abstract         = {
    Deep Learning (DL) has pervaded many areas of computing due to the confluence of the explosive growth of large-scale computing capabilities, availability of datasets, and advances in learning techniques.
    However, the infrastructure that supports DL is still in its early stage, bearing mismatches among the hardware, the software stack, and DL applications.
    On the one hand, despite the emergence of new unique hardware and new use cases, the software stack that abstracts and schedules these hardware resources remains largely unchanged.
    On the other hand, user-defined performance metrics common in DL applications urge better schedulers tailored to the application's specific needs.
    Motivated by the mismatch, this dissertation revisits the system design across the stack, with a focus on the synergy between schedulers and application/system-specific information.

    At the bottom level, the ever-growing adoption of specialized hardware like GPUs poses challenges to efficient usage.
    Due to the lack of operating system arbitration, applications usually assume exclusive access, making the otherwise underutilized resources unusable for other jobs on the same host.
    We therefore design Salus to realize proper efficient GPU sharing.
    It leverages DL applications' specific usage patterns to schedule iterations and manage memory allocations, providing two missing primitives: fast job switching and memory sharing.

    However, even with an efficient execution platform, it is still not trivial to harvest the hardware's full potential for higher-level applications. We investigate two such cases sitting on opposite sides of a model's lifecycle: hyperparameter tuning and inference serving.

    Hyperparameter tuning -- which constitutes a great portion of DL cluster usage given the proliferation of distributed resources in clusters -- generates many small interdependent training trials.
    Existing tuning algorithms are oblivious of advanced execution strategies like intra-GPU sharing and inter-GPU execution, often causing poor resource utilization.
    Hence, we propose Fluid as a generalized hyperparameter tuning execution engine, that coordinates between tuning jobs and cluster resources.
    Fluid schedules training trials in such jobs using a water-filling approach to make the best use of resources at both intra- and inter-GPU granularity to speed up hyperparameter tuning.

    Moving on, inference serving also requires careful scheduling to achieve tight latency guarantees and maintain high utilization.
    Existing serving solutions assume inference execution times to be data-independent and thus highly predictable.
    However, with the rise of dynamic neural networks, data-dependent inferences see higher variance in execution times and become less predictable by a single, point estimation of the true running times.
    With Orloj, we show that treating and modeling inference execution times as probability distributions bring large gains for scheduling inference requests in the presence of SLO constraints.

    In this dissertation, we consider combining application/system-specific information with scheduling design as a means of efficiently supporting new hardware and new DL application use cases.
    Nevertheless, the pursuit of higher efficiency never ends.
    This dissertation tries to lay down the necessary mechanisms with the hope that our crude work may be a basis for further research to better scheduling algorithms and more efficient systems in the DL infrastructure.
  }
}

@PhDThesis{hasanal:dissertation,
  Title                    = {Practical Memory Disaggregation},
  Author                   = {Hasan Al Maruf},
  Institution              = {University of Michigan},
  Year                     = {2023},
  Month                    = {August},

  publist_confkey = {dissertation},
  publist_link = {paper || hasanal-dissertation.pdf},
  Abstract                 = {In today's datacenters, compute and memory resources are tightly-coupled. This causes fleet-wide resource underutilization and increases the total cost of ownership (TCO) for large-scale datacenters. Modern datacenters are embracing a paradigm shift towards disaggregation, where each resource type is decoupled and connected through a network fabric. As memory is the prime resource for high-performance services, it has become an attractive target for disaggregation. Disaggregating memory from compute enables flexibility to scale them independently and better resource utilization. As memory consumes 30-40% of the total rack power and operating cost, proper utilization of stranded resources through disaggregation can save billions of dollars in TCO.

With the advent of ultra-fast networks and coherent interfaces like CXL, disaggregation has become popular over the last few years. There are, however, many open challenges for its practical adoption, including the latency gap between local and remote memory access, resilience, deployability in existing infrastructure, adaptation of heterogeneity in cluster resources, and isolation while maintaining the quality of service. To make memory disaggregation widely adoptable, besides hardware support, we need to provide performant software stacks considering all these challenges so that such systems do not degrade application performance beyond a noticeable margin.

This dissertation proposes a comprehensive solution to address the host-level, network-level, and end-to-end aspects of practical memory disaggregation. Existing memory disaggregation solutions usually use data path components designed for slow disks. As a result, applications experience remote memory access latency significantly higher than that of the underlying low-latency network. To bridge the still-sizeable latency gap between local vs. remote memory access, we design Leap – a prefetching solution for remote memory accesses. At its core, Leap employs an online, majority-based prefetching algorithm, which increases the page cache hit rate. Next, comes the challenge of providing resilience. Relying on memory across multiple machines in a disaggregated cluster makes applications susceptible to a wide variety of uncertainties, such as independent and correlated failures of remote machines, evictions from and corruptions of remote memory, network partitions, etc. Applications also suffer from stragglers or late-arriving remote responses because of the latency variabilities in a large network due to congestion and background traffic. Hydra addresses these issues by enabling a low-latency, low-overhead, and highly available erasure-coded resilient remote memory datapath at single-digit μs tail latency.

For widespread deployability, besides private clouds, we consider public clouds where prevail a set of unique challenges for resource disaggregation across different tenants, including resource harvesting, isolation, and matching. We design Memtrade to enable memory disaggregation on public clouds even in the absence of the latest networking hardware and protocols (e.g., RDMA, CXL). Memtrade allows producer virtual machines (VMs) to lease both their unallocated memory and allocated-but-idle application memory to remote consumer VMs for a limited period of time.

Emerging coherent interfaces like CXL reduce the remote memory access latency to a few hundred nanoseconds. It enables main memory expansion where different memory technologies with varied characteristics can co-exist. Without efficient memory management, however, such heterogeneous tiered-memory systems can significantly degrade performance. We propose a novel OS-level page placement mechanism, TPP, for tiered-memory systems. TPP employs a lightweight mechanism to identify and place hot/cold pages to appropriate memory tiers.

Altogether, this dissertation presents how to enable practical memory disaggregation for next-generation datacenters through performant, resilient, and easily deployable software stacks.}
}


@PhDThesis{fanlai:dissertation,
  Title                    = {Minimalist Systems for Pervasive Machine Learning},
  Author                   = {Fan Lai},
  Institution              = {University of Michigan},
  Year                     = {2023},
  Month                    = {August},

  publist_confkey = {dissertation},
  publist_badge = {Dennis M. Ritchie Award Honorable Mention},
  publist_badge = {Kuck Dissertation Prize},
  publist_badge = {Towner Prize},
  publist_link = {paper || fanlai-dissertation.pdf},
  Abstract                 = {Skyrocketing data volumes, growing hardware capabilities, and the revolution in machine learning (ML) theory have collectively driven the latest leap forward in ML. Despite our hope to realize the next leap with new hardware and a broader range of data, ML development is reaching scaling limits in both realms. First, the exponential surge in ML workload volumes and their complexity far outstrip hardware improvements, leading to hardware resource demands surpassing the sustainable growth of capacity. Second, the mounting volumes of edge data, increasing awareness of user privacy, and tightening government regulations render conventional ML practices, which centralize all data into the cloud, increasingly unsustainable due to escalating costs and scrutiny.

  This dissertation surmounts these resource and data limits using a minimalist approach – reducing complexity and eliminating bloating features – to develop minimalist systems. The thesis provides evidence that by co-designing ML, systems, and networking, we can (1) minimize ML resource demands by removing bloating system execution without compromising ML performance; (2) minimize data collection by effectively offloading ML to the planet-scale data source; and (3) minimize human effort by automatically discovering the sweet spot of ML and system efficiency.

  The minimalist systems developed in this thesis span each stage of ML development, facilitating the transition to the era of pervasive ML. The thesis commences with the data preprocessing stage and introduces a network-aware execution engine called Sol. Sol empowers distributed ML clusters to efficiently perform collaborative data processing over the Internet to minimize data migration. The second and third parts of this thesis optimize the subsequent training stage, by introducing ModelKeeper and AdaEmbed to minimize ML resource demands. ModelKeeper repurposes the weights of previously trained models to warm up model training, reducing the amount of training execution needed. During model training, AdaEmbed automatically identifies the model weights that contribute more to model accuracy and removes less important weights, reducing model size without compromising model accuracy.

  The fourth and fifth parts introduce FedScale and Oort to complement and extend today’s ML training stage and the subsequent deployment stage up to the planetary scale. They enable federated model training and testing across millions of clients at the edge. FedScale supports on-device model execution and integrates Oort to orchestrate clients. At runtime, Oort cherry-picks the clients, who have the data that offers better utility in improving model accuracy and the capability to execute the ML task quickly, to minimize the performance gap between cloud ML and federated ML.}
}

@PhDThesis{yiwenzhg:dissertation,
  Title                    = {Quality of Service for Performance-Critical Cloud Applications},
  Author                   = {Yiwen Zhang},
  Institution              = {University of Michigan},
  Year                     = {2024},
  Month                    = {July},

  publist_confkey          = {dissertation},
  publist_link             = {paper || yiwen-dissertation.pdf},
  publist_abstract         = {Cloud infrastructure continues to scale due to rapid evolvement of both hardware and software technologies in recent years. On the one hand, recent hardware advancement such as accelerators, kernel-bypass networks, and high-speed interconnect brings more powerful computing devices, faster networking equipment, and larger data storage. On the other hand, new software technologies such as computer vision and natural language processing introduce more workloads across datacenters and the edge. As a result, more and more applications from many tenants with different performance requirements must share the compute and network resources to improve resource utilization. Therefore, it is more important than ever to ensure performance-critical applications receive the appropriate level of priority and service quality.

This dissertation aims to build system support for better quality of service (QoS) for performance-critical applications in the cloud. Specifically, we aim to provide guaranteed performance specified by service level objectives (SLOs) for multiple coexisting applications while maximizing system resource utilization. Unfortunately, we observe that existing cloud infrastructure lacks QoS support in multiple critical places including network interface cards (NICs), datacenter fabrics, edge devices and tiered memory systems, each of which requires unique QoS-aware system design to ensure predictable application performance.

To this end, we have built software solutions to provide better QoS in each of the aforementioned areas. First, we built Justitia to provide performance isolation and fairness in the NIC for kernel-bypass networks (KBNs). Justitia overcomes the unique challenges in KBN with several innovations, including split connections with message-level shaping, sender-based resource mediation with receiver-side updates, and passive latency monitoring. Second, we built Aequitas to provide QoS for latency-critical remote procedure calls (RPCs) inside datacenter networks. Aequitas is a distributed sender-driven admission control scheme that uses commodity Weighted-Fair Queuing (WFQ) to guarantee RPC-level SLOs. It enforces cluster-wide RPC latency SLOs via probabilistic downgrading in order to limit the amount of traffic admitted into different QoS levels. Third, we built Vulcan to automatically generate query plans for live ML queries based on their accuracy and end-to-end latency requirements, while minimizing resource consumption across the edge. Vulcan determines the best pipeline, placement, and query configuration by combining several techniques including Bayesian Optimization and memorizing intermediate results of pipeline operators. Finally, we built Mercury, a QoS-aware tiered memory system to provide predictable performance for memory-intensive applications. Mercury proposes a new resource management scheme inside the kernel tailored for tiered memory systems. It leverages a novel admission control and a real-time adaptation algorithm to ensure QoS guarantees for both latency-sensitive and bandwidth-intensive applications. Together, these solutions provide the missing pieces from the edge to the cloud to enable QoS for performance-critical cloud applications.}
}


@article{swan:arxiv22,
	  title={Swan: A Neural Engine for Efficient DNN Training on Smartphone SoCs},
	  author={Sanjay Sri Vallabh Singapuram and Fan Lai and Chuheng Hu and Mosharaf Chowdhury},
	  archiveprefix = {arXiv},
	  eprint= {},
	  url= {https://arxiv.org/abs/2206.04687},
	  year={2022},
	  month={May},
	  publist_confkey = {arXiv:2206.04687},
	  publist_link = {paper || https://arxiv.org/abs/2206.04687},
	  publist_topic = {Systems + AI},
	  publist_topic = {Wide-Area Computing},
	  publist_abstract = { The need to train DNN models on end-user devices (e.g., smartphones) is
		  increasing with the need to improve data privacy and reduce communication overheads.
		    Unlike datacenter servers with powerful CPUs and GPUs, modern smartphones consist
		    of a diverse collection of specialized cores following a system-on-a-chip (SoC)
		    architecture that together perform a variety of tasks. We observe that training
		    DNNs on a smartphone SoC without carefully considering its resource constraints
		    can not only lead to suboptimal training performance but significantly affect
		    user experience as well. In this paper, we present Swan, a neural engine to
		    optimize DNN training on smartphone SoCs without hurting user experience.
	          Extensive large-scale evaluations show that Swan can improve performance by
		    1.2 - 23.3x over the state-of-the-art.
	  }
}


@article{auxo:arxiv22,
	  title={Auxo: Heterogeneity-Mitigating Federated Learning via Scalable Client Clustering},
	  author={Jiachen Liu and Fan Lai and Yinwei Dai and Aditya Akella and Harsha Madhyastha and Mosharaf Chowdhury},
	  archiveprefix = {arXiv},
	  eprint= {},
	  url= {https://arxiv.org/abs/2210.16656},
	  year={2022},
	  month={Oct},
	  publist_confkey = {arXiv:2210.16656},
	  publist_link = {paper || https://arxiv.org/abs/2210.16656},
	  publist_topic = {Systems + AI},
	  publist_topic = {Wide-Area Computing},
	  publist_abstract = {Federated learning (FL) is an emerging machine learning (ML) paradigm that enables heterogeneous edge devices to collaboratively train ML models without revealing their raw data to a logically centralized server. Heterogeneity across participants is a fundamental challenge in FL, both in terms of non-independent and identically distributed(Non-IID) data distributions and variations in device capabilities. Many existing works present point solutions to address issues like slow convergence, low final accuracy, and bias in FL, all stemming from the client heterogeneity.We observe that, in a large population, there exist groups of clients with statistically similar data distributions(cohorts). In this paper, we propose Auxo to gradually identify cohorts among large-scale, low-participation, and resource-constrained FL populations. Auxo then adaptively determines how to train cohort-specific models in order to achieve better model performance and ensure resource efficiency. By identifying cohorts with smaller heterogeneity and performing efficient cohort-based training, our extensive evaluations show that Auxo substantially boosts the state-of-the-art solutions in terms of final accuracy, convergence time, and model bias.
	  }
}




@PhDThesis{jieyou:dissertation,
  Title                    = {Toward Practical Application-Aware Big Data Systems},
  Author                   = {Jie You},
  Institution              = {University of Michigan},
  Year                     = {2022},
  Month                    = {August},
  publist_confkey          = {dissertation},
  publist_link             = {paper || jieyou-dissertation.pdf},
  publist_abstract         = {
    The increasing number of Internet of things (IoT) and other connected devices has led to a surge in the amount of data collected and analyzed.
    Data scientists collect useful insights from these data through data analysis or machine learning, all of which are performed on distributed big data infrastructures. 
    Improving the performance and efficiency of such systems can, therefore, improve user experience and reduce their operating cost. 

    These big data infrastructures abstract away low-level details such as resource allocation and task placement decisions and expose simple APIs for the users.
    Despite simplifying the development process, the decoupling between applications and infrastructures complicates the optimization for end-to-end performance and efficiency metrics, as the contextual details and optimization objectives of applications cannot be expressed using the APIs.
    We observe that system designers often optimize only for common system-level metrics such as throughput and latency, inadvertently ignoring application-level semantics.
    As a result, these best-effort local optimizations do not always improve in application-level performance or efficiency.
    Moreover, the situation is exacerbated as new emerging applications become more intricate and their interactions with the infrastructures become more complex.

    To alleviate the mismatch, we focus on bringing application-awareness into the infrastructures.
    We explore the opportunities of co-optimizing applications and infrastructures for three popular big data use cases: data analytics, transaction processing, and deep learning, and highlight two ways of implementing application-awareness: white-box design and black-box design.
    For data analytics, we present Terra which explicitly passes application-level context into the infrastructures, adopting the white-box design of application-awareness.
    For transaction processing, it is impossible to maintain an accurate model of the performance curve capturing the relationship between request rate and latency, rendering the white-box design infeasible.
    Therefore, we present Kayak with a black-box application-aware design to adaptively arbitrate incoming requests, improving overall throughput and CPU utilization.
    For deep learning, the white-box solution is intractable due to the increasing depth of the neural network.
    To this end, we also present Zeus with a black-box design, adopting a Multi-Armed Bandit algorithm to optimize for faster and more energy-efficient DL training.

    In this dissertation, we demonstrate the adoption of application-awareness as a means of optimizing the end-to-end performance and efficiency of the underlying infrastructures with additional information from the application.
    We discuss two design patterns for implementing application-awareness, along with their tradeoffs, in the context of three popular big data use cases.
    The insights from this dissertation promote the adoption of application-awareness and can help future system researchers build more performant and efficient big data infrastructures.
  }
}

@Article{zeus:arxiv22,
  author        = {Jie You and Jae-Won Chung and Mosharaf Chowdhury},
  journal       = {CoRR},
  title         = {Zeus: Understanding and Optimizing {GPU} Energy Consumption of {DNN} Training},
  year          = {2022},
  month         = {Aug},
  volume        = {abs/2208.06102},
  archiveprefix = {arXiv},
  eprint        = {2208.06102},
  url           = {https://arxiv.org/abs/2208.06102},

  publist_confkey = {arXiv:2208.06102},
  publist_link = {paper || https://arxiv.org/abs/2208.06102},
  publist_link = {code || https://github.com/ml-energy/zeus},
  publist_link = {website || https://ml.energy/zeus},
  publist_topic = {Energy-Efficient Systems},
  publist_topic = {Systems + AI},
  publist_abstract = {
    Training deep neural networks (DNNs) is becoming more and more resource- and energy-intensive every year. Unfortunately, existing works primarily focus on optimizing DNN training for faster completion, often without considering the impact on energy efficiency.

    In this paper, we observe that common practices to improve training performance can often lead to inefficient energy usage. More importantly, we demonstrate that there is a tradeoff between energy consumption and performance optimization. To this end, we propose an optimization framework, Zeus, to navigate this tradeoff by automatically finding optimal job- and GPU-level configurations for recurring DNN training jobs. Zeus uses an online exploration-exploitation approach in conjunction with just-in-time energy profiling, averting the need for expensive offline measurements, while adapting to data drifts over time. Our evaluation shows that Zeus can improve the energy efficiency of DNN training by 15.3%--75.8% for diverse workloads.
  }
}

@article{cdi-e:vldb22,
  author    = {Prakash C. Das and Shivangi Srivastava and Valentin Moskovich and Anmol Chaturvedi and Anant Mittal and Yongqin Xiao and Mosharaf Chowdhury},
  title     = {{CDI-E:} An Elastic Cloud Service for Data Engineering},
  journal   = {Proceedings of the VLDB Endowment},
  volume    = {15},
  number    = {12},
  pages     = {3319--3331},
  year      = {2022},
  month     = {August},

  publist_confkey = {VLDB'22},
  publist_link = {paper || cdi-e-vldb22.pdf},
  publist_abstract = {
    We live in the gilded age of data-driven computing. With public clouds offering virtually unlimited amount of compute and storage, enterprises collecting data about every aspect of their businesses, and advances in analytics and machine learning technologies, data driven decision making is now timely, cost-effective, and therefore, pervasive. Alas, only a handful of power users can wield today’s powerful data engineering tools. For one thing, most solutions require knowledge of specific programming interfaces or libraries. Furthermore, running them requires complex configurations and knowledge of the underlying cloud for cost-effectiveness. 
    
    We decided that a fundamental redesign is in order to democratize data engineering for the masses at cloud scale. The result is Informatica Cloud Data Integration - Elastic (CDI-E). Since the early 1990s, Informatica has been a pioneer and industry leader in building no-code data engineering tools. Non-experts can express complex data engineering tasks using a graphical user interface (GUI). Informatica CDI-E is built to incorporate the simplicity of GUI in the design layer with an elastic and highly scalable runtime to handle data in any format without little to no user input using automated optimizations. Users upload their data to the cloud in any format and can immediately use them in conjunction with their data management and analytics tools of choice using CDI-E GUI. Implementation began in the Spring of 2017, and Informatica CDIE has been generally available since the Summer of 2019. Today, CDI-E is used in production by a growing number of small and large enterprises to make sense of data in arbitrary formats. 
    
    In this paper, we describe the architecture of Informatica CDI-E and its novel no-code data engineering interface. The paper highlights some of the key features of CDI-E: simplicity without loss in productivity and extreme elasticity. It concludes with lessons we learned and an outlook of the future.  }
}

@Article{orloj:arxiv22,
  author        = {Peifeng Yu and Yuqing Qiu and Xin Jin and Mosharaf Chowdhury},
  journal       = {CoRR},
  title         = {{Orloj}: Predictably Serving Unpredictable {DNN}s},
  year          = {2022},
  month         = {Aug},
  volume        = {abs/2209.00159},
  archiveprefix = {arXiv},
  bibsource     = {dblp computer science bibliography, https://dblp.org},
  biburl        = {https://dblp.org/rec/journals/corr/abs-2209-00159.bib},
  eprint        = {2209.00159},
  url           = {https://arxiv.org/abs/2209.00159},

  publist_confkey = {arXiv:2209.00159},
  publist_link = {paper || https://arxiv.org/abs/2209.00159},
  publist_topic = {Systems + AI},
  publist_abstract = {
    Existing DNN serving solutions can provide tight latency SLOs while maintaining high throughput via careful
  scheduling of incoming requests, whose execution times are assumed to be highly predictable and data-independent.
  However, inference requests to emerging dynamic DNNs -- e.g., popular natural language processing (NLP) models and
  computer vision (CV) models that skip layers -- are data-dependent. They exhibit poor performance when served using
  existing solutions because they experience large variance in request execution times depending on the input -- the
  longest request in a batch inflates the execution times of the smaller ones, causing SLO misses in the absence of
  careful batching.

    In this paper, we present Orloj, a dynamic DNN serving system, that captures this variance in dynamic DNNs using
  empirical distributions of expected request execution times, and then efficiently batches and schedules them without
  knowing a request's precise execution time. Orloj significantly outperforms state-of-the-art serving solutions for
  high variance dynamic DNN workloads by 51--80% in finish rate under tight SLO constraints, and over 100% under more
  relaxed SLO settings. For well-studied static DNN workloads, Orloj keeps comparable performance with the
  state-of-the-art.
  }
}

@InProceedings{zeus:nsdi23,
  author    = {Jie You and Jae-Won Chung and Mosharaf Chowdhury},
  booktitle = {USENIX NSDI},
  title     = {Zeus: Understanding and Optimizing {GPU} Energy Consumption of {DNN} Training},
  year      = {2023},

  publist_confkey = {NSDI'23},
  publist_link = {paper || zeus-nsdi23.pdf},
  publist_link = {code || https://github.com/ml-energy/zeus},
  publist_link = {website || https://ml.energy/zeus},
  publist_topic = {Energy-Efficient Systems},
  publist_topic = {Systems + AI},
  publist_abstract = {
    Training deep neural networks (DNNs) is becoming increasingly more resource- and energy-intensive every year. Unfortunately, existing works primarily focus on optimizing DNN training for faster completion, often without considering the impact on energy efficiency.

    In this paper, we observe that common practices to improve training performance can often lead to inefficient energy usage. More importantly, we demonstrate that there is a tradeoff between energy consumption and performance optimization. To this end, we propose Zeus, an optimization framework to navigate this tradeoff by automatically finding optimal job- and GPU-level configurations for recurring DNN training jobs. Zeus uses an online exploration-exploitation approach in conjunction with just-in-time energy profiling, averting the need for expensive offline measurements, while adapting to data drifts over time. Our evaluation shows that Zeus can improve the energy efficiency of DNN training by 15.3%–75.8% for diverse workloads.
  }
}

@InProceedings{modelkeeper:nsdi23,
  author    = {Fan Lai and Yinwei Dai and Harsha V. Madhyastha and Mosharaf Chowdhury},
  booktitle = {USENIX NSDI},
  title     = {{ModelKeeper}: Accelerating {DNN} Training via Automated Training Warmup},
  year      = {2023},

  publist_confkey = {NSDI'23},
  publist_link = {paper || modelkeeper-nsdi23.pdf},
  publist_link = {code || https://github.com/SymbioticLab/ModelKeeper},
  publist_topic = {Systems + AI},
  publist_abstract = {
    With growing deployment of machine learning (ML) models,
    ML developers are training or re-training increasingly more
    deep neural networks (DNNs). They do so to find the most
    suitable model that meets their accuracy requirement while
    satisfying the resource and timeliness constraints of the target
    environment. In large shared clusters, the growing number
    of neural architecture search (NAS) and training jobs often
    result in models sharing architectural similarities with others
    from the same or a different ML developer. However, existing
    solutions do not provide a systematic mechanism to identify
    and leverage such similarities.
    We present ModelKeeper, the first automated training
    warmup system that accelerates DNN training by repurposing previously-trained models in a shared cluster. Our key
    insight is that initializing a training job's model by transforming an already-trained model's weights can jump-start it and
    reduce the total amount of training needed. However, models submitted over time can differ in their architectures and
    accuracy. Given a new model to train, ModelKeeper scalably
    identifies its architectural similarity with previously trained
    models, selects a parent model with high similarity and good
    model accuracy, and performs structure-aware transformation
    of weights to preserve maximal information from the parent
    model during the warmup of new model weights. Our evaluations across thousands of CV and NLP models show that
    ModelKeeper achieves 1.3x-4.3x faster training completion
    with little overhead and no reduction in model accuracy.
  }
}

@Article{dpack:arxiv22,
  author        = {Tholoniat, Pierre and Kostopoulou, Kelly and Chowdhury, Mosharaf and Cidon, Asaf and Geambasu, Roxana and Lécuyer, Mathias and Yang, Junfeng},
  journal       = {CoRR},
  title         = {Packing Privacy Budget Efficiently},
  year          = {2022},
  month         = {Dec},
  volume        = {abs/2212.13228},
  archiveprefix = {arXiv},
  eprint        = {2212.13228},
  url           = {https://arxiv.org/abs/2212.13228},
  publist_confkey = {arXiv:2212.13228},
  publist_link = {paper || https://arxiv.org/abs/2212.13228},
  publist_topic = {Systems + AI},
  publist_abstract = {
Machine learning (ML) models can leak information about users, and differential privacy (DP) provides a rigorous way to bound that leakage under a given budget. This DP budget can be regarded as a new type of compute resource in workloads of multiple ML models training on user data. Once it is used, the DP budget is forever consumed. Therefore, it is crucial to allocate it most efficiently to train as many models as possible. This paper presents the scheduler for privacy that optimizes for efficiency. We formulate privacy scheduling as a new type of multidimensional knapsack problem, called privacy knapsack, which maximizes DP budget efficiency. We show that privacy knapsack is NP-hard, hence practical algorithms are necessarily approximate. We develop an approximation algorithm for privacy knapsack, DPK, and evaluate it on microbenchmarks and on a new, synthetic private-ML workload we developed from the Alibaba ML cluster trace. We show that DPK: (1) often approaches the efficiency-optimal schedule, (2) consistently schedules more tasks compared to a state-of-the-art privacy scheduling algorithm that focused on fairness (1.3-1.7x in Alibaba, 1.0-2.6x in microbenchmarks), but (3) sacrifices some level of fairness for efficiency. Therefore, using DPK, DP ML operators should be able to train more models on the same amount of user data while offering the same privacy guarantee to their users.
  }
}

@Article{flint:arxiv23,
  author        = {Wang, Ewen and Kannan, Ajay and Liang, Yuefeng and Chen, Boyi and Chowdhury, Mosharaf},
  journal       = {CoRR},
  title         = {{FLINT}: A Platform for Federated Learning Integration},
  year          = {2023},
  month         = {Feb},
  volume        = {abs/2302.12862},
  archiveprefix = {arXiv},
  eprint        = {2302.12862},
  url           = {https://arxiv.org/abs/2302.12862},
  publist_confkey = {arXiv:2302.12862},
  publist_link = {paper || https://arxiv.org/abs/2302.12862},
  publist_topic = {Systems + AI},
  publist_topic = {Wide-Area Computing},
  publist_abstract = {
Cross-device federated learning (FL) has been well-studied from algorithmic, system scalability, and training speed perspectives. Nonetheless, moving from centralized training to cross-device FL for millions or billions of devices presents many risks, including performance loss, developer inertia, poor user experience, and unexpected application failures. In addition, the corresponding infrastructure, development costs, and return on investment are difficult to estimate. In this paper, we present a device-cloud collaborative FL platform that integrates with an existing machine learning platform, providing tools to measure real-world constraints, assess infrastructure capabilities, evaluate model training performance, and estimate system resource requirements to responsibly bring FL into production. We also present a decision workflow that leverages the FL-integrated platform to comprehensively evaluate the trade-offs of cross-device FL and share our empirical evaluations of business-critical machine learning applications that impact hundreds of millions of users.
  }
}

@Article{chase:arxiv23,
  author        = {Yang, Zhenning and Meng, Luoxi and Chung, Jae-Won and Chowdhury, Mosharaf},
  journal       = {CoRR},
  title         = {Chasing Low-Carbon Electricity for Practical and Sustainable {DNN} Training},
  year          = {2023},
  month         = {Mar},
  volume        = {abs/2303.02508},
  archiveprefix = {arXiv},
  eprint        = {2303.02508},
  url           = {https://arxiv.org/abs/2303.02508},
  publist_confkey = {arXiv:2303.02508},
  publist_link = {paper || https://arxiv.org/abs/2303.02508},
  publist_topic = {Systems + AI},
  publist_topic = {Energy-Efficient Systems},
  publist_abstract = {
Deep learning has experienced significant growth in recent years, resulting in increased energy consumption and carbon emission from the use of GPUs for training deep neural networks (DNNs). Answering the call for sustainability, conventional solutions have attempted to move training jobs to locations or time frames with lower carbon intensity. However, moving jobs to other locations may not always be feasible due to large dataset sizes or data regulations. Moreover, postponing training can negatively impact application service quality because the DNNs backing the service are not updated in a timely fashion. In this work, we present a practical solution that reduces the carbon footprint of DNN training without migrating or postponing jobs. Specifically, our solution observes real-time carbon intensity shifts during training and controls the energy consumption of GPUs, thereby reducing carbon footprint while maintaining training performance. Furthermore, in order to proactively adapt to shifting carbon intensity, we propose a lightweight machine learning algorithm that predicts the carbon intensity of the upcoming time frame. Our solution, Chase, reduces the total carbon footprint of training ResNet-50 on ImageNet by 13.6\% while only increasing training time by 2.5\%.
  }
}


@InProceedings{egeria:eurosys23,
  author    = {Yiding Wang and Decang Sun and Kai Chen and Fan Lai and Mosharaf Chowdhury},
  booktitle = {EuroSys},
  title     = {{Egeria}: Efficient {DNN} Training with Knowledge-Guided Layer Freezing},
  year      = {2023},
  publist_confkey = {EuroSys'23},
  publist_link = {paper || egeria-eurosys23.pdf},
  publist_topic = {Systems + AI},
  publist_abstract = {
    Training deep neural networks (DNNs) is time-consuming. While most existing solutions try to overlap/schedule computation and communication for efficient training, this paper goes one step further by skipping computing and communication through DNN layer freezing. Our key insight is that the training progress of internal DNN layers differs significantly, and front layers often become well-trained much earlier than deep layers. To explore this, we first introduce the notion of training plasticity to quantify the training progress of internal DNN layers. Then we design Egeria, a knowledge-guided DNN training system that employs semantic knowledge from a reference model to accurately evaluate individual layers’ training plasticity and safely freeze the converged ones, saving their corresponding backward computation and communication. Our reference model is generated on the fly using quantization techniques and runs forward operations asynchronously on available CPUs to minimize the overhead. In addition, Egeria caches the intermediate outputs of the frozen layers with prefetching to further skip the forward computation. Our implementation and testbed experiments with popular vision and language models show that Egeria achieves 19%-43% training speedup w.r.t. the state-of-the-art without sacrificing accuracy.
  }
}

@InProceedings{flint:mlsys23,
  author    = {Ewen Wang and Ajay Kannan and Yuefeng Liang and Boyi Chen and Mosharaf Chowdhury},
  booktitle = {MLSys},
  title     = {{FLINT}: A Platform for Federated Learning Integration},
  year      = {2023},
  publist_confkey = {MLSys'23},
  publist_link = {paper || flint-mlsys23.pdf},
  publist_topic = {Systems + AI},
  publist_topic = {Wide-Area Computing},
  publist_abstract = {
Cross-device federated learning (FL) has been well-studied from algorithmic, system scalability, and training speed perspectives.
Nonetheless, moving from centralized training to cross-device FL for millions or billions of devices presents many risks, including performance loss, developer inertia, poor user experience, and unexpected application failures. 
In addition, the corresponding infrastructure, development costs, and return on investment are difficult to estimate. 
In this paper, we present a device-cloud collaborative FL platform that integrates with an existing machine learning platform, providing tools to measure real-world constraints, assess infrastructure capabilities, evaluate model training performance, and estimate system resource requirements to responsibly bring FL into production.
We also present a decision workflow that leverages the FL-integrated platform to comprehensively evaluate the trade-offs of cross-device FL and share our empirical evaluations of business-critical machine learning applications that impact hundreds of millions of users.
  }
}

@InProceedings{chase:ccai23,
  author        = {Yang, Zhenning and Meng, Luoxi and Chung, Jae-Won and Chowdhury, Mosharaf},
  booktitle     = {Tackling Climate Change with Machine Learning},
  title         = {Chasing Low-Carbon Electricity for Practical and Sustainable {DNN} Training},
  year          = {2023},

  publist_confkey = {CCAI-ICLR'23},
  publist_link = {paper || chase-ccai23.pdf},
  publist_topic = {Systems + AI},
  publist_topic = {Energy-Efficient Systems},
  publist_abstract = {
Deep learning has experienced significant growth in recent years, resulting in increased energy consumption and carbon emission from the use of GPUs for training deep neural networks (DNNs). Answering the call for sustainability, conventional solutions have attempted to move training jobs to locations or time frames with lower carbon intensity. However, moving jobs to other locations may not always be feasible due to large dataset sizes or data regulations. Moreover, postponing training can negatively impact application service quality because the DNNs backing the service are not updated in a timely fashion. In this work, we present a practical solution that reduces the carbon footprint of DNN training without migrating or postponing jobs. Specifically, our solution observes real-time carbon intensity shifts during training and controls the energy consumption of GPUs, thereby reducing carbon footprint while maintaining training performance. Furthermore, in order to proactively adapt to shifting carbon intensity, we propose a lightweight machine learning algorithm that predicts the carbon intensity of the upcoming time frame. Our solution, Chase, reduces the total carbon footprint of training ResNet-50 on ImageNet by 13.6\% while only increasing training time by 2.5\%.
  }
}

@Article{disaggregation-future:arxiv23,
  author        = {Hasan Al Maruf and Mosharaf Chowdhury},
  journal       = {CoRR},
  title         = {Memory Disaggregation: Advances and Open Challenges},
  year          = {2023},
  month         = {May},
  volume        = {abs/2305.03943},
  archiveprefix = {arXiv},
  eprint        = {2305.03943},
  url           = {https://arxiv.org/abs/2305.03943},
  publist_confkey = {arXiv:2305.03943},
  publist_link = {paper || https://arxiv.org/abs/2305.03943},
  publist_topic = {Disaggregation},
  publist_abstract = {
Compute and memory are tightly coupled within each server in traditional datacenters. Large-scale datacenter operators have identified this coupling as a root cause behind fleet-wide resource underutilization and increasing Total Cost of Ownership (TCO). With the advent of ultra-fast networks and cache-coherent interfaces, memory disaggregation has emerged as a potential solution, whereby applications can leverage available memory even outside server boundaries.

This paper summarizes the growing research landscape of memory disaggregation from a software perspective and introduces the challenges toward making it practical under current and future hardware trends. We also reflect on our seven-year journey in the SymbioticLab to build a comprehensive disaggregated memory system over ultra-fast networks. We conclude with some open challenges toward building next-generation memory disaggregation systems leveraging emerging cache-coherent interconnects.
  }
}

@InProceedings{disaggregation-future:hotinfra23,
  author        = {Hasan Al Maruf and Mosharaf Chowdhury},
  title         = {Memory Disaggregation: Open Challenges in the Era of {CXL}},
  booktitle = {HotInfra},
  year          = {2023},
  month         = {June},
  publist_confkey = {HotInfra'23},
  publist_link = {paper || memory-disaggregation-hotinfra23.pdf},
  publist_link = {slides || memory-disaggregation-slides-hotinfra23.pdf},
  publist_topic = {Disaggregation},
  publist_abstract = {
    Compute and memory are tightly coupled within traditional datacenter servers. Large-scale datacenter operators have identified this coupling as a root cause behind fleet-wide resource underutilization and increasing Total Cost of Ownership (TCO). With the advent of ultra-fast networks and cache-coherent interfaces, memory disaggregation has emerged as a potential solution, whereby applications can leverage available memory even outside server boundaries. In this paper, we discuss some open challenges from a software perspective toward building next-generation memory disaggregation systems leveraging emerging cache-coherent interconnects.
  }
}

@article{disaggregation-future:osr23,
  author        = {Hasan Al Maruf and Mosharaf Chowdhury},
  title         = {Memory Disaggregation: Advances and Open Challenges},
  journal       = {ACM SIGOPS Operating Systems Review},
  year          = {2023},
  month         = {June},
  Number        = {1},
  Pages         = {29--37},
  Volume        = {57},
  publist_confkey = {OSR:57(1)},
  publist_link = {paper || https://dl.acm.org/doi/abs/10.1145/3606557.3606562},
  publist_topic = {Disaggregation},
  publist_abstract = {
Compute and memory are tightly coupled within each server in traditional datacenters. Large-scale datacenter operators have identified this coupling as a root cause behind fleetwide resource underutilization and increasing Total Cost of Ownership (TCO). With the advent of ultra-fast networks and cache-coherent interfaces, memory disaggregation has emerged as a potential solution, whereby applications can leverage available memory even outside server boundaries.

This paper summarizes the growing research landscape of memory disaggregation from a software perspective and introduces the challenges toward making it practical under current and future hardware trends. We also reflect on our seven-year journey in the SymbioticLab to build a comprehensive disaggregated memory system over ultra-fast networks. We conclude with some open challenges toward building next-generation memory disaggregation systems leveraging emerging cache-coherent interconnects.  }
}

@InProceedings{adaembed:osdi23,
  author    = {Fan Lai and Wei Zhang and Rui Liu and William Tsai and Xiaohan Wei and Yuxi Hu and Sabin Devkota and Jianyu Huang and Jongsoo Park and Xing Liu and Zeliang Chen and Ellie Wen and Paul Rivera and Jie You and Jason Chen and Mosharaf Chowdhury},
  booktitle = {OSDI},
  title     = {{AdaEmbed}: Adaptive Embedding for Large-Scale Recommendation Models},
  year      = {2023},
  publist_confkey = {OSDI'23},
  publist_link = {paper || adaembed-osdi23.pdf},
  publist_topic = {Systems + AI},
  publist_abstract = {
    Deep learning recommendation models (DLRMs) are using increasingly larger embedding tables to represent categorical sparse features such as video genres. Each embedding row of the table represents the trainable weight vector for a specific instance of that feature. While increasing the number of embedding rows typically improves model accuracy by considering more feature instances, it can lead to larger deployment costs and slower model execution.

    Unlike existing efforts that primarily focus on optimizing DLRMs for the given embedding, we present a complementary system, AdaEmbed, to reduce the size of embeddings needed for the same DLRM accuracy via in-training embedding pruning. Our key insight is that the access patterns and weights of different embeddings are heterogeneous across embedding rows, and dynamically change over the training process, implying varying embedding importance with respect to model accuracy. However, identifying important embeddings and then enforcing pruning for modern DLRMs with up to billions of embeddings (terabytes) is challenging. Given the total embedding size, AdaEmbed considers embeddings with higher runtime access frequencies and larger training gradients to be more important, and it dynamically prunes less important embeddings at scale to automatically determine per-feature embeddings. Our evaluations in industrial settings show that AdaEmbed saves 35-60% embedding size needed in deployment and improves model execution speed by 11-34%, while achieving noticeable accuracy gains.
  }
}


@Article{oobleck:arxiv23,
  author        = {Jang, Insu and Yang, Zhenning and Zhang, Zhen and Jin, Xin and Chowdhury, Mosharaf},
  journal       = {CoRR},
  title         = {Oobleck: Resilient Distributed Training of Large Models Using Pipeline Templates},
  year          = {2023},
  month         = {Sep},
  volume        = {abs/2309.08125},
  archiveprefix = {arXiv},
  eprint        = {2309.08125},
  url           = {https://arxiv.org/abs/2309.08125},
  publist_confkey = {arXiv:2309.08125},
  publist_link = {paper || https://arxiv.org/abs/2309.08125},
  publist_topic = {Systems + AI},
  publist_abstract = {
    Oobleck enables resilient distributed training of large DNN models with guaranteed fault tolerance. It takes a planning-execution co-design approach, where it first generates a set of heterogeneous pipeline templates and instantiates at least f+1 logically equivalent pipeline replicas to tolerate any f simultaneous failures. During execution, it relies on already-replicated model states across the replicas to provide fast recovery. Oobleck provably guarantees that some combination of the initially created pipeline templates can be used to cover all available resources after f or fewer simultaneous failures, thereby avoiding resource idling at all times. Evaluation on large DNN models with billions of parameters shows that Oobleck provides consistently high throughput, and it outperforms state-of-the-art fault tolerance solutions like Bamboo and Varuna by up to 29.6x.
  }
}


@InProceedings{oobleck:sosp23,
  author           = {Jang, Insu and Yang, Zhenning and Zhang, Zhen and Jin, Xin and Chowdhury, Mosharaf},
  booktitle        = {SOSP},
  title            = {Oobleck: Resilient Distributed Training of Large Models Using Pipeline Templates},
  year             = {2023},
  month            = {Oct},
  publist_confkey  = {SOSP'23},
  publist_link     = {paper || oobleck-sosp23.pdf},
  publist_topic    = {Systems + AI},
  publist_badge    = {Artifacts Available},
  publist_badge    = {Artifacts Functional},
  publist_badge    = {Results Reproduced},
  publist_abstract = {
    Oobleck enables resilient distributed training of large DNN models with guaranteed fault tolerance. It takes a planning-execution co-design approach, where it first generates a set of heterogeneous pipeline templates and instantiates at least f+1 logically equivalent pipeline replicas to tolerate any f simultaneous failures. During execution, it relies on already-replicated model states across the replicas to provide fast recovery. Oobleck provably guarantees that some combination of the initially created pipeline templates can be used to cover all available resources after f or fewer simultaneous failures, thereby avoiding resource idling at all times. Evaluation on large DNN models with billions of parameters shows that Oobleck provides consistently high throughput, and it outperforms state-of-the-art fault tolerance solutions like Bamboo and Varuna by up to 29.6x.
  }}



@InProceedings{auxo:socc23,
  author    = {Jiachen Liu and Fan Lai and Yinwei Dai and Aditya Akella and Harsha Madhyastha and Mosharaf Chowdhury},
  booktitle = {SoCC},
  title     = {{Auxo}: Efficient Federated Learning via Scalable Client Clustering},
  year      = {2023},
  publist_confkey = {SoCC'23},
  publist_link = {paper || auxo-socc23.pdf},
  publist_topic = {Systems + AI},
  publist_topic = {Wide-Area Computing},
  publist_abstract = { 
   Federated learning (FL) is an emerging machine learning (ML) paradigm that enables heterogeneous edge devices to collaboratively train ML models without revealing their raw data to a logically centralized server. However, beyond the heterogeneous device capacity, FL participants often exhibit differences in their data distributions, which are not independent and identically distributed (Non-IID). Many existing works present point solutions to address issues like slow convergence, low final accuracy, and bias in FL, all stemming from client heterogeneity.

In this paper, we explore an additional layer of complexity to mitigate such heterogeneity by grouping clients with statistically similar data distributions (cohorts). We propose Auxo to gradually identify such cohorts in large-scale, low-availability, and resource-constrained FL populations. Auxo then adaptively determines how to train cohort-specific models in order to achieve better model performance and ensure resource efficiency. Our extensive evaluations show that, by identifying cohorts with smaller heterogeneity and performing efficient cohort-based training, Auxo boosts various existing FL solutions in terms of final accuracy (2.1%–8.2%), convergence time (up to 2.2x), and model bias (4.8% - 53.8%).
  }
}

@InProceedings{cloudless:hotnets23,
  author    = {Yiming Qiu and Patrick Tser Jern Kon and Jiarong Xing and Yibo Huang and Hongyi Liu and Xinyu Wang and Peng Huang and Mosharaf Chowdhury and Ang Chen},
  booktitle = {HotNets},
  title     = {Simplifying Cloud Management with Cloudless Computing},
  year      = {2023},
  publist_confkey = {HotNets'23},
  publist_link = {paper || cloudless-hotnets23.pdf},
  publist_abstract = { 
    Cloud computing has transformed the IT industry, but managing cloud infrastructures remains a difficult task. We make a case for putting today’s management practices, known as “Infrastructure-as-Code,” on a firmer ground via a principled design. We call this end goal Cloudless Computing: it aims to simplify cloud infrastructure management tasks by supporting them “as-a-service,” analogous to serverless computing that relieves users of the burden of managing server instances. By assisting tenants with these tasks, cloud resources will be presented to their users more readily without the undue burden of complex control. We describe the research problems by examining the typical lifecycle of today’s cloud infrastructure management, and identify places where a cloudless approach will advance the state of the art.
  }
}

@InProceedings{flamingo:distributedml23,
  author    = {Sanjay Sri Vallabh Singapuram and Chuheng Hu and Fan Lai and Chengsong Zhang and Mosharaf Chowdhury},
  booktitle = {DistributedML},
  title     = {Flamingo: A User-Centric System for Fast and Energy-Efficient {DNN} Training on Smartphones},
  year      = {2023},
  publist_confkey = {DistributedML'23},
  publist_topic    = {Systems + AI},
  publist_topic = {Wide-Area Computing},
  publist_link = {paper || flamingo-distributedml23.pdf},
  publist_abstract = { 
Training DNNs on a smartphone system-on-a-chip (SoC) without carefully considering its resource constraints leads to suboptimal training performance and significantly affects user experience. To this end, we present Flamingo, a system for smartphones that optimizes DNN training for time and energy under dynamic resource availability, by scaling parallelism and exploiting compute heterogeneity in real-time. As AI becomes a part of the mainstream smartphone experience, the need to train on-device becomes crucial to fine-tune predictive models while ensuring data privacy. Our experiments show that Flamingo achieves significant improvement in reducing time (12×) and energy (8×) for on-device training, while nearly eliminating detrimental user experience. Extensive large-scale evaluations show that Flamingo can improve end-to-end training performance by 1.2–23.3× and energy efficiency by 1.6–7× over the state-of-the-art.
  }
}

@article{treehouse:eir23,
  author        = {Thomas Anderson and Adam Belay and Mosharaf Chowdhury and Asaf Cidon and Irene Zhang},
  title         = {Treehouse: A Case For Carbon-Aware Datacenter Software},
  journal       = {ACM SIGEnergy Energy Informatics Review},
  year          = {2023},
  month         = {Oct},
  Number        = {3},
  Pages         = {64--70},
  Volume        = {3},
  publist_confkey = {EIR:3(3)},
  publist_link = {paper || https://dl.acm.org/doi/abs/10.1145/3630614.3630626},
  publist_topic = {Energy-Efficient Systems},
  publist_abstract = {
The end of Dennard scaling and the slowing of Moore's Law has put the energy use of datacenters on an unsustainable path. Datacenters are already a significant fraction of worldwide electricity use, with application demand scaling at a rapid rate. We argue that substantial reductions in the carbon intensity of datacenter computing are possible with a software-centric approach: by making energy and carbon visible to application developers on a fine-grained basis, by modifying system APIs to make it possible to make informed trade offs between performance and carbon emissions, and by raising the level of application programming to allow for flexible use of more energy efficient means of compute and storage. We also lay out a research agenda for systems software to reduce the carbon footprint of datacenter computing.}
}

@Article{perseus:arxiv23,
  author        = {Jae-Won Chung and Yile Gu and Insu Jang and Luoxi Meng and Nikhil Bansal and Mosharaf Chowdhury},
  journal       = {CoRR},
  title         = {Perseus: Removing Energy Bloat from Large Model Training},
  year          = {2023},
  month         = {Dec},
  volume        = {abs/2312.06902},
  archiveprefix = {arXiv},
  eprint        = {2312.06902},
  url           = {https://arxiv.org/abs/2312.06902},

  publist_confkey = {arXiv:2312.06902},
  publist_link = {paper || https://arxiv.org/abs/2312.06902},
  publist_link = {code || https://github.com/ml-energy/zeus},
  publist_link = {website || https://ml.energy/zeus/perseus},
  publist_topic = {Energy-Efficient Systems},
  publist_topic = {Systems + AI},
  publist_abstract = {
Training large AI models on numerous GPUs consumes a massive amount of energy. We observe that not all energy consumed during training directly contributes to end-to-end training throughput, and a significant portion can be removed without slowing down training, which we call energy bloat.

In this work, we identify two independent sources of energy bloat in large model training, intrinsic and extrinsic, and propose Perseus, a unified optimization framework that mitigates both. Perseus obtains the "iteration time–energy" Pareto frontier of any large model training job using an efficient iterative graph cut-based algorithm and schedules energy consumption of its forward and backward computations across time to remove intrinsic and extrinsic energy bloat. Evaluation on large models like GPT-3 and Bloom shows that Perseus reduces energy consumption of large model training by up to 30\%, enabling savings otherwise unobtainable before.
  }
}

@InProceedings{perseus:sosp24,
  author           = {Jae-Won Chung and Yile Gu and Insu Jang and Luoxi Meng and Nikhil Bansal and Mosharaf Chowdhury},
  booktitle        = {SOSP},
  title            = {Reducing Energy Bloat in Large Model Training},
  year             = {2024},
  month            = {Nov},
  publist_confkey  = {SOSP'24},
  publist_link     = {paper || perseus-sosp24.pdf},
  publist_link     = {code || https://github.com/ml-energy/zeus},
  publist_link     = {website || https://ml.energy/zeus/research_overview/perseus},
  publist_topic    = {Energy-Efficient Systems},
  publist_topic    = {Systems + AI},
  publist_badge    = {Artifacts Available},
  publist_badge    = {Artifacts Functional},
  publist_badge    = {Results Reproduced},
  publist_abstract = {
Training large AI models on numerous GPUs consumes a massive amount of energy, making power delivery one of the largest limiting factors in building and operating datacenters for AI workloads. However, we observe that not all energy consumed during training directly contributes to end-to-end throughput; a significant portion can be removed without slowing down training. We call this portion energy bloat.

In this work, we identify two independent sources of energy bloat in large model training and propose Perseus, a training system that mitigates both. To do this, Perseus obtains the time–energy tradeoff frontier of a large model training job using an efficient graph cut-based algorithm, and schedules computation energy consumption across time to reduce both types of energy bloat. Evaluation on large models, including GPT-3 and Bloom, shows that Perseus reduces the energy consumption of large model training by up to 30% without any throughput loss or hardware modification.
  }}

@Article{llm-survey:arxiv23,
  author        = {Zhongwei Wan and Xin Wang and Che Liu and Samiul Alam and Yu Zheng and Zhongnan Qu and Shen Yan and Yi Zhu and Quanlu Zhang and Mosharaf Chowdhury and Mi Zhang},
  journal       = {CoRR},
  title         = {Efficient Large Language Models: A Survey},
  year          = {2023},
  month         = {Dec},
  volume        = {abs/2312.03863},
  archiveprefix = {arXiv},
  eprint        = {2312.03863},
  url           = {https://arxiv.org/abs/2312.03863},

  publist_confkey = {arXiv:2312.03863},
  publist_link = {paper || https://arxiv.org/abs/2312.03863},
  publist_link = {website || https://github.com/AIoT-MLSys-Lab/Efficient-LLMs-Survey},
  publist_topic = {Systems + AI},
  publist_abstract = {
Large Language Models (LLMs) have demonstrated remarkable capabilities in important tasks such as natural language understanding, language generation, and complex reasoning and have the potential to make a substantial impact on our society. Such capabilities, however, come with the considerable resources they demand, highlighting the strong need to develop effective techniques for addressing their efficiency challenges. In this survey, we provide a systematic and comprehensive review of efficient LLMs research. We organize the literature in a taxonomy consisting of three main categories, covering distinct yet interconnected efficient LLMs topics from model-centric, data-centric, and framework-centric perspective, respectively. We have also created a GitHub repository where we compile the papers featured in this survey, and will actively maintain this repository and incorporate new research as it emerges. We hope our survey can serve as a valuable resource to help researchers and practitioners gain a systematic understanding of the research developments in efficient LLMs and inspire them to contribute to this important and exciting field.  }
}

@Article{venn:arxiv23,
  author        = {Jiachen Liu and Fan Lai and Ding Ding and Yiwen Zhang and Mosharaf Chowdhury},
  journal       = {CoRR},
  title         = {Venn: Resource Management Across Federated Learning Jobs},
  year          = {2023},
  month         = {Dec},
  volume        = {abs/2312.08298},
  archiveprefix = {arXiv},
  eprint        = {2312.08298},
  url           = {https://arxiv.org/abs/2312.08298},
  publist_confkey = {arXiv:2312.08298},
  publist_link = {paper || https://arxiv.org/abs/2312.08298}, 
  publist_topic = {Systems + AI},
  publist_topic = {Wide-Area Computing},
  publist_abstract = {In recent years, federated learning (FL) has emerged as a promising approach for machine learning (ML) and data science across distributed edge devices. With the increasing popularity of FL, resource contention between multiple FL jobs training on the same device population is increasing as well. Scheduling edge resources among multiple FL jobs is different from GPU scheduling for cloud ML because of the ephemeral nature and planetary scale of participating devices as well as the overlapping resource requirements of diverse FL jobs. Existing resource managers for FL jobs opt for random assignment of devices to FL jobs for simplicity and scalability, which leads to poor performance. 

In this paper, we present Venn, an FL resource manager, that efficiently schedules ephemeral, heterogeneous devices among many FL jobs, with the goal of reducing their average job completion time (JCT). Venn formulates the Intersection Resource Scheduling (IRS) problem to identify complex resource contention among multiple FL jobs. Then, Venn proposes a contention-aware scheduling heuristic to minimize the average scheduling delay. Furthermore, it proposes a resource-aware device-to-job matching heuristic that focuses on optimizing response collection time by mitigating stragglers. Our evaluation shows that, compared to the state-of-the-art FL resource managers, Venn improves the average JCT by up to 1.88X.
  }
}

@Article{crosslayer-energy:arxiv24,
  author        = {Jae-Won Chung and Mosharaf Chowdhury},
  journal       = {CoRR},
  title         = {Toward Cross-Layer Energy Optimizations in Machine Learning Systems},
  year          = {2024},
  month         = {Apr},
  volume        = {abs/2404.06675},
  archiveprefix = {arXiv},
  eprint        = {2404.06675},
  url           = {https://arxiv.org/abs/2404.06675},
  publist_confkey = {arXiv:2404.06675},
  publist_link = {paper || https://arxiv.org/abs/2404.06675}, 
  publist_topic = {Systems + AI},
  publist_topic = {Energy-Efficient Systems},
  publist_abstract = {The enormous energy consumption of machine learning (ML) and generative AI workloads shows no sign of waning, taking a toll on operating costs, power delivery, and environmental sustainability. Despite a long line of research on energy-efficient hardware, we found that software plays a critical role in ML energy optimization through two recent works: Zeus and Perseus. This is especially true for large language models (LLMs) because their model sizes and, therefore, energy demands are growing faster than hardware efficiency improvements. Therefore, we advocate for a cross-layer approach for energy optimizations in ML systems, where hardware provides architectural support that pushes energy-efficient software further, while software leverages and abstracts the hardware to develop techniques that bring hardware-agnostic energy-efficiency gains.
  }
}

@inproceedings{crosslayer-energy:eecs24,
  author        = {Jae-Won Chung and Nishil Talati and Mosharaf Chowdhury},
  booktitle     = {Energy-Efficient Computing for Science Workshop},
  title         = {Toward Cross-Layer Energy Optimizations in {AI} Systems},
  year          = {2024},

  publist_confkey = {EECS'24},
  publist_link = {paper || crosslayer-energy-eecs24.pdf}, 
  publist_topic = {Systems + AI},
  publist_topic = {Energy-Efficient Systems},
  publist_abstract = {The ``AI for Science, Energy, and Security'' report from DOE outlines a significant focus on developing and optimizing artificial intelligence workflows for a foundational impact on a broad range of DOE missions. With the pervasive usage of artificial intelligence (AI) and machine learning (ML) tools and techniques, their energy efficiency is likely to become the gating factor toward adoption. This is because generative AI (GenAI) models are massive energy hogs: for instance, training a 200-billion parameter large language model (LLM) at Amazon is estimated to have taken 11.9 GWh, which is enough to power more than a thousand average U.S. households for a year. Inference consumes even more energy, because a model trained once serve millions. Given this scale, high energy efficiency is key to addressing the power delivery problem of constructing and operating new supercomputers and datacenters specialized for AI workloads. In that regard, we outline software- and architecture-level research challenges and opportunities, setting the stage for creating cross-layer energy optimizations in AI systems.
  }
}

@Article{fedtrans:arxiv24,
  author    = {Yuxuan Zhu and Jiachen Liu and Mosharaf Chowdhury and Fan Lai},
  title     = {{FedTrans}: Efficient Federated Learning Over Heterogeneous Clients via Model Transformation},
  year          = {2024},
  month         = {Apr},
  volume        = {abs/2404.13515},
  archiveprefix = {arXiv},
  eprint        = {2404.13515},
  url           = {https://arxiv.org/abs/2404.13515},
  publist_confkey = {arXiv:2404.13515},
  publist_link = {paper || https://arxiv.org/abs/2404.13515}, 
  publist_topic = {Systems + AI},
  publist_topic = {Wide-Area Computing},
  publist_abstract = {
Federated learning (FL) aims to train machine learning (ML) models across potentially millions of edge client devices. Yet, training and customizing models for FL clients is notoriously challenging due to the heterogeneity of client data, device capabilities, and the massive scale of clients, making individualized model exploration prohibitively expensive. State-of-the-art FL solutions personalize a globally trained model or concurrently train multiple models, but they often incur suboptimal model accuracy and huge training costs.

In this paper, we introduce FedTrans, a multi-model FL training framework that automatically produces and trains high-accuracy, hardware-compatible models for individual clients at scale. FedTrans begins with a basic global model, identifies accuracy bottlenecks in model architectures during training, and then employs model transformation to derive new models for heterogeneous clients on the fly. It judiciously assigns models to individual clients while performing soft aggregation on multi-model updates to minimize total training costs. Our evaluations using realistic settings show that FedTrans improves individual client model accuracy by 13% while slashing training costs by 4x over state-of-the-art solutions.
  }
} 

@InProceedings{fedtrans:mlsys24,
  author    = {Yuxuan Zhu and Jiachen Liu and Mosharaf Chowdhury and Fan Lai},
  booktitle = {MLSys},
  title     = {{FedTrans}: Efficient Federated Learning via Multi-Model Transformation},
  year      = {2024},	
  publist_confkey = {MLSys'24},
  publist_topic = {Systems + AI},
  publist_link = {paper || fedtrans-mlsys24.pdf},
  publist_topic = {Wide-Area Computing},
  publist_abstract = {
Federated learning (FL) aims to train machine learning (ML) models across potentially millions of edge client devices. Yet, training and customizing models for FL clients is notoriously challenging due to the heterogeneity of client data, device capabilities, and the massive scale of clients, making individualized model exploration prohibitively expensive. State-of-the-art FL solutions personalize a globally trained model or concurrently train multiple models, but they often incur suboptimal model accuracy and huge training costs.

In this paper, we introduce FedTrans, a multi-model FL training framework that automatically produces and trains high-accuracy, hardware-compatible models for individual clients at scale. FedTrans begins with a basic global model, identifies accuracy bottlenecks in model architectures during training, and then employs model transformation to derive new models for heterogeneous clients on the fly. It judiciously assigns models to individual clients while performing soft aggregation on multi-model updates to minimize total training costs. Our evaluations using realistic settings show that FedTrans improves individual client model accuracy by 13% while slashing training costs by 4x over state-of-the-art solutions.
  }
} 

@Article{andes:arxiv24,
  author        = {Jiachen Liu and Zhiyu Wu and Jae-Won Chung and Fan Lai and Myungjin Lee and Mosharaf Chowdhury},
  journal       = {CoRR},
  title         = {{Andes}: Defining and Enhancing Quality-of-Experience in {LLM}-Based Text Streaming Services},
  year          = {2024},
  month         = {Apr},
  volume        = {abs/2404.16283},
  archiveprefix = {arXiv},
  eprint        = {2404.16283},
  url           = {https://arxiv.org/abs/2404.16283},
  publist_confkey = {arXiv:2404.16283},
  publist_link = {paper || https://arxiv.org/abs/2404.16283}, 
  publist_topic = {Systems + AI},
  publist_abstract = {The advent of large language models (LLMs) has transformed text-based services, enabling capabilities ranging from real-time translation to AI-driven chatbots. However, existing serving systems primarily focus on optimizing server-side aggregate metrics like token generation throughput, ignoring individual user experience with streamed text. As a result, under high and/or bursty load, a significant number of users can receive unfavorable service quality or poor Quality-of-Experience (QoE). 
  
  In this paper, we first formally define QoE of text streaming services, where text is delivered incrementally and interactively to users, by considering the end-to-end token delivery process throughout the entire interaction with the user. Thereafter, we propose Andes, a QoE-aware serving system that enhances user experience for LLM-enabled text streaming services. At its core, Andes strategically allocates contended GPU resources among multiple requests over time to optimize their QoE. Our evaluations demonstrate that, compared to the state-of-the-art LLM serving systems like vLLM, Andes improves the average QoE by up to 3.2X under high request rate, or alternatively, it attains up to 1.6X higher request rate while preserving high QoE.
    }
}

@InProceedings{memstrata:osdi24,
  author    = {Yuhong Zhong and Daniel S. Berger and Carl Waldspurger and Ryan Wee and Ishwar Agarwal and Rajat Agarwal and Frank Hady and Karthik Kumar and Mark D. Hill and Mosharaf Chowdhury and Asaf Cidon},
  booktitle = {OSDI},
  title     = {Managing Memory Tiers with {CXL} in Virtualized Environments},
  year      = {2024},	
  publist_confkey = {OSDI'24},
  publist_topic = {Disaggregation},
  publist_link = {paper || memstrata-osdi24.pdf},
  publist_abstract = {
Cloud providers seek to deploy CXL-based memory to increase aggregate memory capacity, reduce costs, and lower carbon emissions. However, CXL accesses incur higher latency than local DRAM. Existing systems use software to manage data placement across memory tiers at page granularity. Cloud providers are reluctant to deploy software-based tiering due to high overheads in virtualized environments. Hardware-based memory tiering could place data at cacheline granularity, mitigating these drawbacks. However, hardware is oblivious to application-level performance.

We propose combining hardware-managed tiering with software-managed performance isolation to overcome the pitfalls of either approach. We introduce Intel® Flat Memory Mode, the first hardware-managed tiering system for CXL. Our evaluation on a full-system prototype demonstrates that it provides performance close to regular DRAM, with no more than 5% degradation for more than 82% of workloads. Despite such small slowdowns, we identify two challenges that can still degrade performance by up to 34% for “outlier” workloads: (1) memory contention across tenants, and (2) intra-tenant contention due to conflicting access patterns.

To address these challenges, we introduce Memstrata, a lightweight multi-tenant memory allocator. Memstrata employs page coloring to eliminate inter-VM contention. It improves performance for VMs with access patterns that are sensitive to hardware tiering by allocating them more local DRAM using an online slowdown estimator. In multi-VM experiments on prototype hardware, Memstrata is able to identify performance outliers and reduce their degradation from above 30% to below 6%, providing consistent performance across a wide range of workloads.
  }
} 

@article{fed-ensemble:ieee-tase,
  author        = {Naichen Shi and Fan Lai and Raed Al Kontar and Mosharaf Chowdhury},
  title         = {Fed-ensemble: Ensemble Models in Federated Learning for Improved Generalization and Uncertainty Quantification},
  journal       = {IEEE Transactions on Automation Science and Engineering},
  year          = {2024},
  month         = {July},
  volume        = {21},
  number        = {3},
  pages         = {2792-2803},
  publist_confkey = {IEEE TASE:21(3)},
  publist_link = {paper || https://doi.org/10.1109/TASE.2023.3269639},
  publist_topic = {Wide-Area Computing},
  publist_abstract = {
The increase in the computational power of edge devices has opened up the possibility of processing some of the data at the edge and distributing model learning. This paradigm is often called federated learning (FL), where edge devices exploit their local computational resources to train models collaboratively. Though FL has seen recent success, it is unclear how to characterize uncertainties in FL predictions. In this paper, we propose Fed-ensemble : a simple approach that brings model ensembling to FL. Instead of aggregating local models to update a single global model, Fed-ensemble uses random permutations to update a group of $K$ models and then obtains predictions through model averaging. Fed-ensemble can be readily utilized within established FL methods and does not impose a computational overhead compared with single-model methods. Empirical results show that our model has superior performance over several FL algorithms on a wide range of data sets and excels in heterogeneous settings often encountered in FL applications. Also, by carefully choosing client-dependent weights in the inference stage, Fed-ensemble becomes personalized and yields even better performance. Theoretically, we show that predictions on new data from all $K$ models belong to the same predictive posterior distribution under a neural tangent kernel regime. This result, in turn, sheds light on the generalization advantages of model averaging and justifies the uncertainty quantification capability. We also illustrate that Fed-ensemble has an elegant Bayesian interpretation. Note to Practitioners —provides an algorithm that extracts a set of $K$ solutions without imposing any additional communication overhead in FL. Given multiple solutions, Fed-ensemble can be exploited to personalize inference as well as quantify uncertainty. Such capabilities may be beneficial within multiple practical systems that require uncertainty-aware decision-making. Further, Fed-ensemble may be useful for model validation and hypothesis testing.
}
}

@article{llm-survey:tmlr,
  author        = {Zhongwei Wan and Xin Wang and Che Liu and Samiul Alam and Yu Zheng and Jiachen Liu and Zhongnan Qu and Shen Yan and Yi Zhu and Quanlu Zhang and Mosharaf Chowdhury and Mi Zhang},
  title         = {Efficient Large Language Models: A Survey},
  journal       = {Transactions on Machine Learning Research},
  year          = {2024},
  month         = {May},
  publist_confkey = {TMLR},
  publist_link = {paper || https://openreview.net/forum?id=bsCCJHbO8A},
  publist_topic = {Systems + AI},
  publist_abstract = {
Large Language Models (LLMs) have demonstrated remarkable capabilities in important tasks such as natural language understanding and language generation, and thus have the potential to make a substantial impact on our society. Such capabilities, however, come with the considerable resources they demand, highlighting the strong need to develop effective techniques for addressing their efficiency challenges. In this survey, we provide a systematic and comprehensive review of efficient LLMs research. We organize the literature in a taxonomy consisting of three main categories, covering distinct yet interconnected efficient LLMs topics from model-centric, data-centric, and framework-centric perspective, respectively. We have also created a GitHub repository where we organize the papers featured in this survey at https://github.com/AIoT-MLSys-Lab/Efficient-LLMs-Survey. We will actively maintain the repository and incorporate new research as it emerges. We hope our survey can serve as a valuable resource to help researchers and practitioners gain a systematic understanding of efficient LLMs research and inspire them to contribute to this important and exciting field.}
}

@article{pyxis:tpds,
  author        = {Sheng Qi and Chao Jin and Mosharaf Chowdhury and Zhenming Liu and Xuanzhe Liu and Xin Jin},
  title         = {Pyxis: Scheduling Mixed Tasks in Disaggregated Datacenters},
  journal       = {IEEE Transactions on Parallel and Distributed Systems},
  year          = {2024},
  month         = {September},
  volume        = {35},
  number        = {9},
  pages         = {1536-1550},
  publist_confkey = {IEEE TPDS:35(9)},
  publist_link = {paper || https://doi.org/10.1109/TPDS.2024.3418620},
  publist_topic = {Disaggregation},
  publist_abstract = {
Disaggregating compute from storage is an emerging trend in cloud computing. Effectively utilizing resources in both compute and storage pool is the key to high performance. The state-of-the-art scheduler provides optimal scheduling decisions for workloads with homogeneous tasks. However, cloud applications often generate a mix of tasks with diverse compute and IO characteristics, resulting in sub-optimal performance for existing solutions. We present Pyxis, a system that provides optimal scheduling decisions for mixed workloads in disaggregated datacenters with theoretical guarantees. Pyxis is capable of maximizing overall throughput while meeting latency SLOs. Pyxis decouples the scheduling of different tasks. Our insight is that the optimal solution has an “all-or-nothing” structure that can be captured by a single turning point in the spectrum of tasks. Based on task characteristics, the turning point partitions the tasks either all to storage nodes or all to compute nodes (none to storage nodes). We theoretically prove that the optimal solution has such a structure, and design an online algorithm with sub-second convergence. We implement a prototype of Pyxis. Experiments on CloudLab with various synthetic and application workloads show that Pyxis improves the throughput by 3–21× over the state-of-the-art solution.}
}

@InProceedings{dpack:eurosys25,
  author    = {Pierre Tholoniat and Kelly Kostopoulou and Mosharaf Chowdhury and Asaf Cidon and Roxana Geambasu and Mathias Lécuyer and Junfeng Yang},
  booktitle = {EuroSys},
  title     = {{DPack}: Efficiency-Oriented Privacy Budget Scheduling},
  year      = {2025},	
  publist_confkey = {EuroSys'25},
  publist_topic = {Systems + AI},
  publist_badge   = {Artifacts Available},
  publist_badge   = {Artifacts Functional},
  publist_link = {paper || dpack-eurosys25.pdf},
  publist_abstract = {
Machine learning (ML) models can leak information about users, and differential privacy (DP) provides a rigorous way to bound that leakage under a given budget. This DP budget can be regarded as a new type of compute resource in workloads of multiple ML models training on user data. Once it is used, the DP budget is forever consumed. Therefore, it is crucial to allocate it most efficiently to train as many models as possible. This paper presents a scheduler for privacy that optimizes for efficiency. We formulate privacy scheduling as a new type of multidimensional knapsack problem, called privacy knapsack, which maximizes DP budget efficiency. We show that privacy knapsack is NP-hard, hence practical algorithms are necessarily approximate. We develop an approximation algorithm for privacy knapsack, DPack, and evaluate it on microbenchmarks and on a new, synthetic private-ML workload we developed from the Alibaba ML cluster trace. We show that DPack: (1) often approaches the efficiency-optimal schedule, (2) consistently schedules more tasks compared to a state-of-the-art privacy scheduling algorithm that focused on fairness (1.3–1.7X in Alibaba, 1.0–2.6X in microbenchmarks), but (3) sacrifices some level of fairness for efficiency. Therefore, using DPack, DP ML operators should be able to train more models on the same amount of user data while offering the same privacy guarantee to their users.  }
} 

@InProceedings{openinfra:hotinfra24,
  author = {Jiaheng Lu and Yunming Xiao and Shmeelok Chakraborty and Silvery Fu and Yoon Sung Ji and Ang Chen and Mosharaf Chowdhury and Nalini Rao and Sylvia Ratnasamy and Xinyu Wang},
  title         = {{OpenInfra}: A Co-simulation Framework for the Infrastructure Nexus},
  booktitle = {HotInfra},
  year          = {2024},
  month         = {November},
  publist_confkey = {HotInfra'24},
  publist_link = {paper || openinfra-hotinfra24.pdf},
  publist_link = {code || https://github.com/JhengLu/OpenInfra},
  publist_topic = {Energy-Efficient Systems},
  publist_abstract = {
    Critical infrastructures like datacenters, power grids, and water systems are interdependent, forming complex "infrastructure nexuses" that require co-optimization for efficiency, resilience, and sustainability. We present OpenInfra, a co-simulation framework designed to model these interdependencies by integrating domain-specific simulators for datacenters, power grids, and cooling systems but focusing on stitching them together for end-to-end experimentation. OpenInfra enables seamless integration of diverse simulators and flexible configuration of infrastructure interactions. Our evaluation demonstrates its ability to simulate large-scale infrastructure dynamics, including 7,392 servers over 100+ hours.
  }
}

@InProceedings{infa-finops:bigdata24,
  author    = {Atam Prakash Agrawal and Anant Mittal and Shivangi Srivastava and Michael Brevard and Valentin Moskovich and Mosharaf Chowdhury},
  title     = {{INFA-FinOps} for Cloud Data Integration},
  booktitle = {BigData},
  year          = {2024},
  month         = {December},
  publist_confkey = {BigData'24},
  publist_link = {paper || infa-finops-bigdata24.pdf},
  publist_abstract = {
Over the past decade, businesses have migrated to the cloud for its simplicity, elasticity, and resilience. Cloud ecosystems offer a variety of computing and storage options, enabling customers to choose configurations that maximize productivity. However, determining the right configuration to minimize cost while maximizing performance is challenging, as workloads vary and cloud offerings constantly evolve. Many businesses are overwhelmed with choice overload and often end up making suboptimal choices that lead to inflated cloud spending and/or poor performance.

In this paper, we describe INFA-FinOps, an automated system that helps Informatica customers strike a balance between cost efficiency and meeting SLAs for Informatica Advanced Data Integration (aka CDI-E) workloads. We first describe common workload patterns observed in CDI-E customers and show how INFA-FinOps selects optimal cloud resources and configurations for each workload, adjusting them as workloads and cloud ecosystems change. It also makes recommendations for actions that require user review or input. Finally, we present performance benchmarks on various enterprise use cases and conclude with lessons learned and potential future enhancements.
}
}


@Article{mercury:arxiv24,
  author    = {Jiaheng Lu and Yiwen Zhang and Hasan Al Maruf and Minseo Park and Yunxuan Tang and Fan Lai and Mosharaf Chowdhury},
  title     = {{Mercury}: {QoS-Aware} Tiered Memory System},
  year          = {2024},
  month         = {Dec},
  volume        = {abs/2412.08938},
  archiveprefix = {arXiv},
  eprint        = {2412.08938},
  url           = {https://arxiv.org/abs/2412.08938},
  publist_confkey = {arXiv:2412.08938},
  publist_link = {paper || https://arxiv.org/abs/2412.08938},
  publist_topic = {Disaggregation},
  publist_abstract = {
Memory tiering has received wide adoption in recent years as an effective solution to address the increasing memory demands of memory-intensive workloads. However, existing tiered memory systems often fail to meet service-level objectives (SLOs) when multiple applications share the system because they lack Quality-of-Service (QoS) support. Consequently, applications suffer severe performance drops due to local memory contention and memory bandwidth interference.

In this paper, we present Mercury, a QoS-aware tiered memory system that ensures predictable performance for coexisting memory-intensive applications with different SLOs. Mercury enables per-tier page reclamation for application-level resource management and uses a proactive admission control algorithm to satisfy SLOs via per-tier memory capacity allocation and intra- and inter-tier bandwidth interference mitigation. It reacts to dynamic requirement changes via real-time adaptation. Extensive evaluations show that Mercury improves application performance by up to 53.4% and 20.3% compared to TPP and Colloid, respectively.
  }
}

@InProceedings{iac-eval:neuripsdb24,
  author    = {Patrick TJ Kon and Jiachen Liu and Yiming Qiu and Weijun Fan and Ting He and Lei Lin and Haoran Zhang and Owen M. Park and George Sajan Elengikal and Yuxin Kang and Ang Chen and Mosharaf Chowdhury and Myungjin Lee and Xinyu Wang},
  title     = {{IaC-Eval}: A code generation benchmark for Infrastructure-as-Code programs},
  year      = {2024},
  booktitle = {NeurIPS D\&B},
  publist_topic = {Systems + AI},
  publist_confkey = {NeurIPS'24 D&B},
  publist_link = {paper || iac-eval-neuripsdb24.pdf},
  publist_link = {code || https://github.com/autoiac-project/iac-eval},
  publist_abstract = {
  Infrastructure-as-Code (IaC), an important component of cloud computing, allows the definition of cloud infrastructure in high-level programs. However, developing IaC programs is challenging, complicated by factors that include the burgeoning complexity of the cloud ecosystem (e.g., diversity of cloud services and workloads), and the relative scarcity of IaC-specific code examples and public repositories. While large language models (LLMs) have shown promise in general code generation and could potentially aid in IaC development, no benchmarks currently exist for evaluating their ability to generate IaC code. We present IaC-Eval, a first step in this research direction. IaC-Eval's dataset includes 458 human-curated scenarios covering a wide range of popular AWS services, at varying difficulty levels. Each scenario mainly comprises a natural language IaC problem description and an infrastructure intent specification. The former is fed as user input to the LLM, while the latter is a general notion used to verify if the generated IaC program conforms to the user's intent; by making explicit the problem's requirements that can encompass various cloud services, resources and internal infrastructure details. Our in-depth evaluation shows that contemporary LLMs perform poorly on IaC-Eval, with the top-performing model, GPT-4, obtaining a pass@1 accuracy of 19.36%. In contrast, it scores 86.6% on EvalPlus, a popular Python code generation benchmark, highlighting a need for advancements in this domain. We open-source the IaC-Eval dataset and evaluation framework at https://github.com/autoiac-project/iac-eval to enable future research on LLM-based IaC code generation.}
}

@Article{mordal:arxiv25,
  author          = {Shiqi He and Insu Jang and Mosharaf Chowdhury},
  title           = {{Mordal}: Automated Pretrained Model Selection for Vision Language Models},
  year            = {2025},
  month           = {Feb},
  volume          = {abs/2502.00241},
  archivePrefix   = {arXiv},
  eprint          = {2502.00241},
  url             = {https://arxiv.org/abs/2502.00241},
  publist_confkey = {arXiv:2502.00241},
  publist_link    = {paper || https://arxiv.org/abs/2502.00241},
  publist_topic   = {Systems + AI},
  publist_abstract = {
    Incorporating multiple modalities into large language models (LLMs) is a powerful way to enhance their understanding of non-textual data, enabling them to perform multimodal tasks. Vision language models (VLMs) form the fastest growing category of multimodal models because of their many practical use cases, including in healthcare, robotics, and accessibility. Unfortunately, even though different VLMs in the literature demonstrate impressive visual capabilities in different benchmarks, they are handcrafted by human experts; there is no automated framework to create task-specific multimodal models.
    
    We introduce Mordal, an automated multimodal model search framework that efficiently finds the best VLM for a user-defined task without manual intervention. Mordal achieves this both by reducing the number of candidates to consider during the search process and by minimizing the time required to evaluate each remaining candidate. Our evaluation shows that Mordal can find the best VLM for a given problem using up to 8.9×-11.6× lower GPU hours than grid search. In the process of our evaluation, we have also discovered new VLMs that outperform their state-of-the-art counterparts.
  }
}

@Article{curie:arxiv25,
  author          = {Patrick Tser Jern Kon and Jiachen Liu and Qiuyi Ding and Yiming Qiu and Zhenning Yang and Yibo Huang and Jayanth Srinivasa and Myungjin Lee and Mosharaf Chowdhury and Ang Chen},
  title           = {{Curie}: Toward Rigorous and Automated Scientific Experimentation with {AI} Agents},
  year            = {2025},
  month           = {Feb},
  volume          = {abs/2502.16069},
  archivePrefix   = {arXiv},
  eprint          = {2502.16069},
  url             = {https://arxiv.org/abs/2502.16069},
  publist_confkey = {arXiv:2502.16069},
  publist_link    = {paper || https://arxiv.org/abs/2502.16069},
  publist_link    = {code || https://github.com/Just-Curieous/Curie},
  publist_link    = {blog || https://www.just-curieous.com},
  publist_topic   = {Systems + AI},
  publist_abstract = {
Scientific experimentation, a cornerstone of human progress, demands rigor in reliability, methodical control, and interpretability to yield meaningful results. Despite the growing capabilities of large language models (LLMs) in automating different aspects of the scientific process, automating rigorous experimentation remains a significant challenge. To address this gap, we propose Curie, an AI agent framework designed to embed rigor into the experimentation process through three key components: an intra-agent rigor module to enhance reliability, an inter-agent rigor module to maintain methodical control, and an experiment knowledge module to enhance interpretability. To evaluate Curie, we design a novel experimental benchmark composed of 46 questions across four computer science domains, derived from influential research papers, and widely adopted open-source projects. Compared to the strongest baseline tested, we achieve a 3.4× improvement in correctly answering experimental questions. Curie is open-sourced at https://github.com/Just-Curieous/Curie.
  }
}

@Article{cornstarch:arxiv25,
  author          = {Insu Jang and Runyu Lu and Nikhil Bansal and Ang Chen and Mosharaf Chowdhury},
  title           = {Cornstarch: Distributed Multimodal Training Must Be Multimodality-Aware},
  year            = {2025},
  month           = {March},
  volume          = {abs/2503.11367},
  archivePrefix   = {arXiv},
  eprint          = {2503.11367},
  url             = {https://arxiv.org/abs/2503.11367},
  publist_confkey = {arXiv:2503.11367},
  publist_link    = {paper || https://arxiv.org/abs/2503.11367},
  publist_link    = {code || https://github.com/cornstarch-org/Cornstarch},
  publist_topic   = {Systems + AI},
  publist_abstract = {
Multimodal large language models (MLLMs) extend the capabilities of large language models (LLMs) by combining heterogeneous model architectures to handle diverse modalities like images and audio. However, this inherent heterogeneity in MLLM model structure and data types makes makeshift extensions to existing LLM training frameworks unsuitable for efficient MLLM training.
In this paper, we present Cornstarch, the first general-purpose distributed MLLM training framework. Cornstarch facilitates modular MLLM construction, enables composable parallelization of constituent models, and introduces MLLM-specific optimizations to pipeline and context parallelism for efficient distributed MLLM training. Our evaluation shows that Cornstarch outperforms state-of-the-art solutions by up to 1.57x in terms of training throughput.
  }
}

@Article{ai-eval-framework:arxiv25,
  author          = {Sarah Jabbour and Trenton Chang and Anindya Das Antar and Joseph Peper and Insu Jang and Jiachen Liu and Jae-Won Chung and Shiqi He and Michael Wellman and Bryan Goodman and Elizabeth Bondi-Kelly and Kevin Samy and Rada Mihalcea and Mosharaf Chowhury and David Jurgens and Lu Wang},
  title           = {Evaluation Framework for {AI} Systems in the Wild},
  year            = {2025},
  month           = {April},
  volume          = {abs/2504.16778},
  archivePrefix   = {arXiv},
  eprint          = {2504.16778},
  url             = {https://arxiv.org/abs/2504.16778},
  publist_confkey = {arXiv:2504.16778},
  publist_link    = {paper || https://arxiv.org/abs/2504.16778},
  publist_topic   = {Systems + AI},
  publist_abstract = {
Generative AI (GenAI) models have become vital across industries, yet current evaluation methods have not adapted to their widespread use. Traditional evaluations often rely on benchmarks and fixed datasets, frequently failing to reflect real-world performance, which creates a gap between lab-tested outcomes and practical applications. This white paper proposes a comprehensive framework for how we should evaluate real-world GenAI systems, emphasizing diverse, evolving inputs and holistic, dynamic, and ongoing assessment approaches. The paper offers guidance for practitioners on how to design evaluation methods that accurately reflect real-time capabilities, and provides policymakers with recommendations for crafting GenAI policies focused on societal impacts, rather than fixed performance numbers or parameter sizes. We advocate for holistic frameworks that integrate performance, fairness, and ethics and the use of continuous, outcome-oriented methods that combine human and automated assessments while also being transparent to foster trust among stakeholders. Implementing these strategies ensures GenAI models are not only technically proficient but also ethically responsible and impactful.
  }
}

@InProceedings{venn:mlsys25,
  author          = {Jiachen Liu and Fan Lai and Ding Ding and Yiwen Zhang and Mosharaf Chowdhury},
  title           = {Venn: Resource Management for Collaborative Learning Jobs},
  booktitle       = {MLSys},  
  year            = {2025},
  publist_link    = {paper || venn-mlsys25.pdf},
  publist_link    = {code || https://github.com/SymbioticLab/Venn},
  publist_confkey = {MLSys'25},
  publist_topic   = {Systems + AI},
  publist_topic   = {Wide-Area Computing},
  publist_badge   = {Artifacts Available},
  publist_badge   = {Artifacts Functional},
  publist_abstract = {
    In recent years, collaborative learning (CL) has emerged as a promising approach for machine learning (ML) and data science across distributed edge devices. As the deployment of CL jobs increases, they inevitably contend for limited resources. However, efficient resource scheduling in this context is challenging because of the ephemeral nature and resource heterogeneity of devices, coupled with the overlapping resource requirements of diverse CL jobs. Existing resource managers often assign devices to CL jobs randomly for simplicity and scalability, but this approach compromises job efficiency.

    In this paper, we present Venn, a CL resource manager that efficiently schedules ephemeral, heterogeneous devices among multiple CL jobs to reduce the average job completion time (JCT). Venn formulates the Intersection Resource Scheduling (IRS) problem to identify complex resource contention among multiple CL jobs. It then proposes a contention-aware scheduling heuristic to minimize the average scheduling delay. Furthermore, it proposes a resource-aware device-to-job matching heuristic to optimize response collection time by mitigating stragglers. Our evaluation shows that, compared to the state-of-the-art CL resource managers, Venn improves the average JCT by up to 1.88x. The code is available at {https://github.com/SymbioticLab/Venn}.
  }
}

@article{mlenergy-benchmark:arxiv25,
  title={The {ML.ENERGY} Benchmark: Toward Automated Inference Energy Measurement and Optimization},
  author={Jae-Won Chung and Jiachen Liu and Jeff J. Ma and Ruofan Wu and Oh Jun Kweon and Yuxuan Xia and Zhiyu Wu and Mosharaf Chowdhury},
  archiveprefix = {arXiv},
  eprint= {2505.06371},
  url= {https://arxiv.org/abs/2505.06371},
  year={2025},
  month={May},
  publist_confkey = {arXiv:2505.06371},
  publist_link = {paper || https://arxiv.org/abs/2505.06371},
  publist_link = {code || https://github.com/ml-energy/benchmark},
  publist_link = {website || https://ml.energy/leaderboard},
  publist_topic = {Energy-Efficient Systems},
  publist_topic = {Systems + AI},
  publist_abstract = {
    As the adoption of Generative AI in real-world services grow explosively, \emph{energy} has emerged as a critical bottleneck resource. However, energy remains a metric that is often overlooked, under-explored, or poorly understood in the context of building ML systems. We present the ML.ENERGY Benchmark, a benchmark suite and tool for measuring inference energy consumption under realistic service environments, and the corresponding ML.ENERGY Leaderboard, which have served as a valuable resource for those hoping to understand and optimize the energy consumption of their generative AI services. In this paper, we explain four key design principles for benchmarking ML energy we have acquired over time, and then describe how they are implemented in the ML.ENERGY Benchmark. We then highlight results from the latest iteration of the benchmark, including energy measurements of 40 widely used model architectures across 6 different tasks, case studies of how ML design choices impact energy consumption, and how automated optimization recommendations can lead to significant (sometimes more than 40\%) energy savings without changing what is being computed by the model. The ML.ENERGY Benchmark is open-source and can be easily extended to various customized models and application scenarios.
  }
}

@Article{expbench:arxiv25,
  author          = {Patrick Tser Jern Kon and Jiachen Liu and Xinyi Zhu and Qiuyi Ding and Jingjia Peng and Jiarong Xing and Yibo Huang and Yiming Qiu and Jayanth Srinivasa and Myungjin Lee and Mosharaf Chowdhury and Matei Zaharia and Ang Chen},
  title           = {{EXP-Bench}: Can {AI} Conduct {AI} Research Experiments?},
  year            = {2025},
  month           = {Jun},
  volume          = {abs/2505.24785},
  archivePrefix   = {arXiv},
  eprint          = {2505.24785},
  url             = {https://arxiv.org/abs/2505.24785},
  publist_confkey = {arXiv:2505.24785},
  publist_link    = {paper || https://arxiv.org/abs/2505.24785},
  publist_link    = {code || https://github.com/Just-Curieous/Curie/tree/main/benchmark/exp_bench},
  publist_link    = {blog || https://www.just-curieous.com/machine-learning/research/2025-06-11-exp-bench-can-ai-conduct-ai-research-experiments.html},
  publist_topic   = {Systems + AI},
  publist_abstract = {
	Automating AI research holds immense potential for accelerating scientific progress, yet current AI agents struggle with the complexities of rigorous, end-to-end experimentation. We introduce EXP-Bench, a novel benchmark designed to systematically evaluate AI agents on complete research experiments sourced from influential AI publications. Given a research question and incomplete starter code, EXP-Bench challenges AI agents to formulate hypotheses, design and implement experimental procedures, execute them, and analyze results. To enable the creation of such intricate and authentic tasks with high-fidelity, we design a semi-autonomous pipeline to extract and structure crucial experimental details from these research papers and their associated open-source code. With the pipeline, EXP-Bench curated 461 AI research tasks from 51 top-tier AI research papers. Evaluations of leading LLM-based agents, such as OpenHands and IterativeAgent on EXP-Bench demonstrate partial capabilities: while scores on individual experimental aspects such as design or implementation correctness occasionally reach 20-35%, the success rate for complete, executable experiments was a mere 0.5%. By identifying these bottlenecks and providing realistic step-by-step experiment procedures, EXP-Bench serves as a vital tool for future AI agents to improve their ability to conduct AI research experiments. EXP-Bench is open-sourced at https://github.com/Just-Curieous/Curie/tree/main/benchmark/exp_bench.
  }
}

@PhDThesis{amberljc:dissertation,
  author          = {Jiachen Liu},
  title           = {User-Centric Machine Learning Systems},
  year            = {2025},
  month           = {June},
  institution     = {University of Michigan},

  publist_link    = {paper || amberljc-dissertation.pdf},
  publist_confkey = {dissertation},
  publist_abstract = {
      Over the past five years, artificial intelligence (AI) has evolved from a specialized technology confined to large corporations and research labs into a ubiquitous tool integrated into everyday life. While AI extends its reach beyond niche domains to individual users across diverse contexts, the widespread adoption has given rise to new needs for machine learning (ML) systems to balance user-centric experiences—such as real-time responsiveness, accessibility and personalization—with system efficiency, including operational cost and resource utilization.
      However, designing such systems is complex due to diverse AI workloads—spanning conversational services, collaborative learning, and large-scale training—as well as the heterogeneous resources, ranging from cloud data centers to resource-constrained edge devices. My research addresses these challenges to achieve these dual objectives through a set of design principles centered on a sophisticated resource scheduler with a server-client co-design paradigm.

      Our contributions are threefold. First, we propose Andes to address the critical need for real-time responsiveness in LLM-backed conversational AI by introducing the concept of QoE tailored for such text streaming service. Our server-side token-level scheduling algorithm dynamically prioritizes token generation based on user-centric metrics, while a co-designed client-side token buffer smooths the streaming experience. This approach significantly improves user experience during peak demand and achieves substantial GPU resource savings.

      Second, we propose Auxo to deliver personalized AI services to a diverse set of end users through scalable collaborative learning. We propose a novel client-clustering mechanism that adapts to statistical data heterogeneity and resource constraints, complemented by a cohort affinity mechanism that empowers clients to join preferred groups while preserving privacy. This approach improves the personalized model performance, adapting to varying needs and contexts of end users.

      Third, we propose Venn, to handle escalating demand for efficient resource sharing in multi-job collaborative learning environments. Our resource scheduler resolves complex resource contention proactively and introduces a novel job offer abstraction that allows client resources to identify eligible jobs based on their local resources. This significantly reduces job completion times and improves resource efficiency.
  }
}

@Article{tetriserve:arxiv25,
  author          = {Runyu Lu and Shiqi He and Wenxuan Tan and Shenggui Li and Ruofan Wu and Jeff J. Ma and Ang Chen and Mosharaf Chowdhury},
  title           = {{TetriServe}: Efficient {DiT} Serving for Heterogeneous Image Generation},
  year            = {2025},
  month           = {Oct},
  volume          = {abs/2510.01565},
  archivePrefix   = {arXiv},
  eprint          = {2510.01565},
  url             = {https://arxiv.org/abs/2510.01565},
  publist_confkey = {arXiv:2510.01565},
  publist_link    = {paper || https://arxiv.org/abs/2510.01565},
  publist_topic   = {Systems + AI},
  publist_abstract = {
  Diffusion Transformer (DiT) models excel at generating high-quality images through iterative denoising steps, but serving them under strict Service Level Objectives (SLOs) is challenging due to their high computational cost, particularly at large resolutions. Existing serving systems use fixed degree sequence parallelism, which is inefficient for heterogeneous workloads with mixed resolutions and deadlines, leading to poor GPU utilization and low SLO attainment. In this paper, we propose step-level sequence parallelism to dynamically adjust the parallel degree of individual requests according to their deadlines. We present TetriServe, a DiT serving system that implements this strategy for highly efficient image generation. Specifically, TetriServe introduces a novel round-based scheduling mechanism that improves SLO attainment: (1) discretizing time into fixed rounds to make deadline-aware scheduling tractable, (2) adapting parallelism at the step level and minimize GPU hour consumption, and (3) jointly packing requests to minimize late completions. Extensive evaluation on state-of-the-art DiT models shows that TetriServe achieves up to 32% higher SLO attainment compared to existing solutions without degrading image quality.
  }
}

@InProceedings{tetriserve:asplos26,
  author          = {Runyu Lu and Shiqi He and Wenxuan Tan and Shenggui Li and Ruofan Wu and Jeff J. Ma and Ang Chen and Mosharaf Chowdhury},
  booktitle       = {ASPLOS},
  title           = {{TetriServe}: Efficiently Serving Mixed {DiT} Workloads},
  year            = {2026},
  month           = {March},
  publist_confkey = {ASPLOS'26},
  publist_link    = {paper || tetriserve-asplos26.pdf},
  publist_topic   = {Systems + AI},
  publist_link    = {code || https://github.com/DiT-Serving/TetriServe},
  publist_abstract = {
  Diffusion Transformer (DiT) models excel at generating high-quality images through iterative denoising steps, but serving them under strict Service Level Objectives (SLOs) is challenging due to their high computational cost, particularly at larger resolutions. Existing serving systems use fixed-degree sequence parallelism, which is inefficient for heterogeneous workloads with mixed resolutions and deadlines, leading to poor GPU utilization and low SLO attainment. In this paper, we propose step-level sequence parallelism to dynamically adjust the degree of parallelism of individual requests according to their deadlines. We present TetriServe, a DiT serving system that implements this strategy for highly efficient image generation. Specifically, TetriServe introduces a novel round-based scheduling mechanism that improves SLO attainment by (1) discretizing time into fixed rounds to make deadline-aware scheduling tractable, (2) adapting parallelism at the step level and minimizing GPU hour consumption, and (3) jointly packing requests to minimize late completions. Extensive evaluation on state-of-the-art DiT models shows that TetriServe achieves up to 32% higher SLO attainment compared to existing solutions without degrading image quality.
  }
}

@InProceedings{mlenergy-benchmark:neuripsdb25,
  title           = {The {ML.ENERGY} Benchmark: Toward Automated Inference Energy Measurement and Optimization}, 
  author          = {Jae-Won Chung and Jeff J. Ma and Ruofan Wu and Jiachen Liu and Oh Jun Kweon and Yuxuan Xia and Zhiyu Wu and Mosharaf Chowdhury},
  year            = {2025},
  month           = {Dec},
  booktitle       = {NeurIPS D\&B},
  publist_topic   = {Systems + AI},
  publist_topic   = {Energy-Efficient Systems},
  publist_confkey = {NeurIPS'25 D&B},
  publist_link    = {paper || mlenergy-benchmark-neuripsdb25.pdf},
  publist_link    = {code || https://github.com/ml-energy/benchmark},
  publist_link    = {website || https://ml.energy/leaderboard},
  publist_badge   = {Spotlight},
  publist_abstract = {
    As the adoption of Generative AI in real-world services grow explosively, energy has emerged as a critical bottleneck resource. However, energy remains a metric that is often overlooked, under-explored, or poorly understood in the context of building ML systems. We present the ML.ENERGY Benchmark, a benchmark suite and tool for measuring inference energy consumption under realistic service environments, and the corresponding ML.ENERGY Leaderboard, which have served as a valuable resource for those hoping to understand and optimize the energy consumption of their generative AI services. In this paper, we explain four key design principles for benchmarking ML energy we have acquired over time, and then describe how they are implemented in the ML.ENERGY Benchmark. We then highlight results from the early 2025 iteration of the benchmark, including energy measurements of 40 widely used model architectures across 6 different tasks, case studies of how ML design choices impact energy consumption, and how automated optimization recommendations can lead to significant (sometimes more than 40\%) energy savings without changing what is being computed by the model. The ML.ENERGY Benchmark is open-source and can be easily extended to various customized models and application scenarios.
  }
}

@InProceedings{rdx:hotnets25,
  author    = {Yibo Huang and Yiming Qiu and Daqian Ding and Patrick Tser Jern Kon and Yiwen Zhang and Yuzhou Mao and Archit Bhatnagar and Mosharaf Chowdhury and Srinivas Devadas and Jiarong Xing and Ang Chen},
  booktitle = {HotNets},
  title     = {Remote Direct Code Execution},
  year      = {2025},
  publist_confkey = {HotNets'25},
  publist_link = {paper || rdx-hotnets25.pdf},
  publist_topic = {Datacenter Networking},
  publist_abstract = { 
    We make a case for remote direct code execution (RDX)—a vision that elevates the power of RDMA from memory access to code execution. Concretely, we target runtime extension frameworks such as Wasm filters, BPF programs, and UDF functions, where RDX enables an agent-less architecture that unlocks capabilities such as fast extension injection, update consistency guarantees, and minimal resource contentions. We envision a technical roadmap for RDX around a new CodeFlow abstraction, encompassing programming remote extensions, exposing management stubs, remotely validating and JIT compiling code, seamlessly linking code to local context, managing remote extension state, and synchronizing code to targets. The case studies and initial results demonstrate the end-to-end feasibility of RDX, and its potential to spark the next wave of RDMA innovations.
  }
}

@Article{branchandbrowse:arxiv25,
  author        = {Shiqi He and Yue Cui and Xinyu Ma and Yaliang Li and Bolin Ding and Mosharaf Chowdhury},
  title         = {{Branch-and-Browse}: Efficient and Controllable Web Exploration with Tree-Structured Reasoning and Action Memory},
  year          = {2025},
  month         = {Oct},
  volume        = {abs/2510.19838},
  archiveprefix = {arXiv},
  eprint        = {2510.19838},
  url           = {https://arxiv.org/abs/2510.19838},
  publist_confkey = {arXiv:2510.19838},
  publist_link = {paper || https://arxiv.org/abs/2510.19838},
  publist_topic = {Systems + AI},
  publist_abstract = {
Autonomous web agents powered by large language models (LLMs) show strong potential for performing goal-oriented tasks such as information retrieval, report generation, and online transactions. These agents mark a key step toward practical embodied reasoning in open web environments. However, existing approaches remain limited in reasoning depth and efficiency: vanilla linear methods fail at multi-step reasoning and lack effective backtracking, while other search strategies are coarse-grained and computationally costly. We introduce Branch-and-Browse, a fine-grained web agent framework that unifies structured reasoning-acting, contextual memory, and efficient execution. It (i) employs explicit subtask management with tree-structured exploration for controllable multi-branch reasoning, (ii) bootstraps exploration through efficient web state replay with background reasoning, and (iii) leverages a page action memory to share explored actions within and across sessions. On the WebArena benchmark, Branch-and-Browse achieves a task success rate of 35.8% and reduces execution time by up to 40.4% relative to state-of-the-art methods. These results demonstrate that Branch-and-Browse is a reliable and efficient framework for LLM-based web agents.
  }
} 

@Article{sphinx:arxiv25,
  author          = {Yuchen Xia and Souvik Kundu and Mosharaf Chowdhury and Nishil Talati},
  title           = {Sphinx: Efficiently Serving Novel View Synthesis using Regression-Guided Selective Refinement},
  year            = {2025},
  month           = {Nov},
  volume          = {abs/2511.18672},
  archivePrefix   = {arXiv},
  eprint          = {2511.18672},
  url             = {https://arxiv.org/abs/2511.18672},
  publist_confkey = {arXiv:2511.18672},
  publist_link    = {paper || https://arxiv.org/abs/2511.18672},
  publist_topic   = {Systems + AI},
  publist_abstract = {
Novel View Synthesis (NVS) is the task of generating new images of a scene from viewpoints that were not part of the original input. Diffusion-based NVS can generate high-quality, temporally consistent images, however, remains computationally prohibitive. Conversely, regression-based NVS offers suboptimal generation quality despite requiring significantly lower compute; leaving the design objective of a high-quality, inference-efficient NVS framework an open challenge. To close this critical gap, we present Sphinx, a training-free hybrid inference framework that achieves diffusion-level fidelity at a significantly lower compute. Sphinx proposes to use regression-based fast initialization to guide and reduce the denoising workload for the diffusion model. Additionally, it integrates selective refinement with adaptive noise scheduling, allowing more compute to uncertain regions and frames. This enables Sphinx to provide flexible navigation of the performance-quality trade-off, allowing adaptation to latency and fidelity requirements for dynamically changing inference scenarios. Our evaluation shows that Sphinx achieves an average 1.8x speedup over diffusion model inference with negligible perceptual degradation of less than 5%, establishing a new Pareto frontier between quality and latency in NVS serving.
  }
}

@Article{cornserve:arxiv25,
  author        = {Jeff J. Ma and Jae-Won Chung and Jisang Ahn and Yizhuo Liang and Akshay Jajoo and Myungjin Lee and Mosharaf Chowdhury},
  journal       = {CoRR},
  title         = {Cornserve: Efficiently Serving Any-to-Any Multimodal Models},
  year          = {2025},
  month         = {Dec},
  volume        = {abs/2512.14098},
  archiveprefix = {arXiv},
  eprint        = {2512.14098},
  url           = {https://arxiv.org/abs/2512.14098},

  publist_confkey = {arXiv:2512.14098},
  publist_link = {paper || https://arxiv.org/abs/2512.14098},
  publist_link = {code || https://github.com/cornserve-ai/cornserve},
  publist_link = {website || https://cornserve.ai},
  publist_topic = {Systems + AI},
  publist_abstract = {
We present Cornserve, an efficient online serving system for an emerging class of multimodal models called Any-to-Any models. Any-to-Any models accept combinations of text and multimodal data (e.g., image, video, audio) as input and also generate combinations of text and multimodal data as output, introducing request type, computation path, and computation scaling heterogeneity in model serving.

Cornserve allows model developers to describe the computation graph of generic Any-to-Any models, which consists of heterogeneous components such as multimodal encoders, autoregressive models like Large Language Models (LLMs), and multimodal generators like Diffusion Transformers (DiTs). Given this, Cornserve's planner automatically finds an optimized deployment plan for the model, including whether and how to disaggregate the model into smaller components based on model and workload characteristics. Cornserve's distributed runtime then executes the model per the plan, efficiently handling Any-to-Any model heterogeneity during online serving. Evaluations show that Cornserve can efficiently serve diverse Any-to-Any models and workloads, delivering up to 3.81x throughput improvement and up to 5.79x tail latency reduction over existing solutions.
  }
}

@article{sigdgx:cacm25,
  author = {Chen, Ang and Ratnasamy, Sylvia and Alizadeh, Mohammad and Chowdhury, Mosharaf and Guikema, Seth and Huang, Ryan and Jaganathann, Suresh and Kerkez, Branko and Lee, Edward and Low, Steven and Mao, Morley and Mathieu, Johanna and Reiter, Michael and Wang, Xinyu and Cerf, Vinton},
  title = {Resilient Infrastructures via Digital Unification},
  journal = {Communications of the ACM},
  year = {2025},
  month = {Dec},
  volume = {69},
  number = {1},
  pages = {38–40},

  publist_confkey = {CACM:69(1)},
  publist_link = {paper || https://dl.acm.org/doi/full/10.1145/3736713},
}

@Article{kareus:arxiv26,
  author          = {Ruofan Wu and Jae-Won Chung and Mosharaf Chowdhury},
  title           = {{Kareus}: Joint Reduction of Dynamic and Static Energy in Large Model Training},
  year            = {2026},
  month           = {Jan},
  volume          = {abs/2601.17654},
  archivePrefix   = {arXiv},
  eprint          = {2601.17654},
  url             = {https://arxiv.org/abs/2601.17654},
  publist_confkey = {arXiv:2601.17654},
  publist_link    = {paper || https://arxiv.org/abs/2601.17654},
  publist_topic   = {Systems + AI},
  publist_topic   = {Energy-Efficient Systems},
  publist_abstract = {
The computing demand of AI is growing at an unprecedented rate, but energy supply is not keeping pace. As a result, energy has become an expensive, contended resource that requires explicit management and optimization. Although recent works have made significant progress in large model training optimization, they focus only on a single aspect of energy consumption: dynamic or static energy.

We find that fine-grained kernel scheduling and frequency scaling jointly and interdependently impact both dynamic and static energy consumption. Based on this finding, we design Kareus, a training system that pushes the time--energy tradeoff frontier by optimizing both aspects. Kareus decomposes the intractable joint optimization problem into local, partition-based subproblems. It then uses a multi-pass multi-objective optimization algorithm to find execution schedules that push the time--energy tradeoff frontier. Compared to the state of the art, Kareus reduces training energy by up to 28.3% at the same training time, or reduces training time by up to 27.5% at the same energy consumption.
  }
}

@article{mlenergy-benchmark-v3:arxiv26,
  title={Where Do the Joules Go? Diagnosing Inference Energy Consumption},
  author={Jae-Won Chung and Ruofan Wu and Jeff J. Ma and Mosharaf Chowdhury},
  archiveprefix = {arXiv},
  eprint= {2601.22076},
  url= {https://arxiv.org/abs/2601.22076},
  year={2026},
  month={Jan},
  publist_confkey = {arXiv:2601.22076},
  publist_link = {paper || https://arxiv.org/abs/2601.22076},
  publist_link = {code || https://github.com/ml-energy/benchmark},
  publist_link = {website || https://ml.energy/leaderboard},
  publist_topic = {Energy-Efficient Systems},
  publist_topic = {Systems + AI},
  publist_abstract = {
Energy is now a critical ML computing resource. While measuring energy consumption and observing trends is a valuable first step, accurately understanding and diagnosing why those differences occur is crucial for optimization. To that end, we begin by presenting a large-scale measurement study of inference time and energy across the generative AI landscape with 46 models, 7 tasks, and 1,858 different configurations on NVIDIA H100 and B200 GPUs. Our empirical findings span order-of-magnitude variations: LLM task type can lead to 25x energy differences, video generation sometimes consumes more than 100x the energy of images, and GPU utilization differences can result in 3--5x energy differences. Based on our observations, we present a framework for reasoning about the underlying mechanisms that govern time and energy consumption. The essence is that time and energy are determined by latent metrics like memory and utilization, which are in turn affected by various factors across the algorithm, software, and hardware layers. Our framework also extends directly to throughput per watt, a critical metric for power-constrained datacenters.
  }
}

@Article{gputogrid:arxiv26,
  author          = {Zhirui Liang and Jae-Won Chung and Mosharaf Chowdhury and Jiasi Chen and Vladimir Dvorkin},
  title           = {{GPU-to-Grid}: Voltage Regulation via GPU Utilization Control},
  year            = {2026},
  month           = {Feb},
  volume          = {abs/2602.05116},
  archivePrefix   = {arXiv},
  eprint          = {2602.05116},
  url             = {https://arxiv.org/abs/2602.05116},
  publist_confkey = {arXiv:2602.05116},
  publist_link    = {paper || https://arxiv.org/abs/2602.05116},
  publist_topic   = {Energy-Efficient Systems},
  publist_topic   = {Systems + AI},
  publist_abstract = {
While the rapid expansion of data centers poses challenges for power grids, it also offers new opportunities as potentially flexible loads. Existing power system research often abstracts data centers as aggregate resources, while computer system research primarily focuses on optimizing GPU energy efficiency and largely ignores the grid impacts of optimized GPU power consumption. To bridge this gap, we develop a GPU-to-Grid framework that couples device-level GPU control with power system objectives. We study distribution-level voltage regulation enabled by flexibility in LLM inference, using batch size as a control knob that trades off the voltage impacts of GPU power consumption against inference latency and token throughput. We first formulate this problem as an optimization problem and then realize it as an online feedback optimization controller that leverages measurements from both the power grid and GPU systems. Our key insight is that reducing GPU power consumption alleviates violations of lower voltage limits, while increasing GPU power mitigates violations near upper voltage limits in distribution systems; this runs counter to the common belief that minimizing GPU power consumption is always beneficial to power grids.
  }
}

@InProceedings{mordal:iclr26,
  author          = {Shiqi He and Insu Jang and Mosharaf Chowdhury},
  booktitle       = {ICLR},
  title           = {{Mordal}: Automated Pretrained Model Selection for Vision Language Models},
  year            = {2026},
  month           = {April},
  publist_confkey = {ICLR'26},
  publist_link    = {paper || mordal-iclr26.pdf},
  publist_topic   = {Systems + AI},
  publist_abstract = {
  Incorporating multiple modalities into large language models (LLMs) is a powerful way to enhance their understanding of non-textual data, enabling them to perform multimodal tasks.
  Vision language models (VLMs) form the fastest growing category of multimodal models because of their many practical use cases, including in healthcare, robotics, and accessibility.
  Unfortunately, even though different VLMs in the literature demonstrate impressive visual capabilities in different benchmarks, they are handcrafted by human experts; there is no automated framework to create task-specific multimodal models.
  
  We introduce Mordal, an automated multimodal model search framework that efficiently finds the best VLM for a user-defined task without manual intervention.
  Mordal achieves this both by reducing the number of candidates to consider during the search process and by minimizing the time required to evaluate each remaining candidate.
  Our evaluation shows that Mordal can find the best VLM for a given problem using $8.9\times$--$11.6\times$ lower GPU hours than grid search.
  We have also discovered that Mordal achieves about 69\% higher weighted Kendall’s $\tau$ on average than the state-of-the-art model selection method across diverse tasks.
  }
}

@InProceedings{expbench:iclr26,
  author          = {Patrick Tser Jern Kon and Qiuyi Ding and Jiachen Liu and Xinyi Zhu and Jingjia Peng and Jiarong Xing and Yibo Huang and Yiming Qiu and Jayanth Srinivasa and Myungjin Lee and Mosharaf Chowdhury and Matei Zaharia and Ang Chen},
  booktitle       = {ICLR},
  title           = {{EXP-BENCH}: Can {AI} Conduct {AI} Research Experiments?},
  year            = {2026},
  month           = {April},
  publist_confkey = {ICLR'26},
  publist_link    = {paper || expbench-iclr26.pdf},
  publist_topic   = {Systems + AI},
  publist_abstract = {
Automating AI research holds immense potential for accelerating scientific progress, yet current AI agents struggle with the complexities of rigorous, end-to-end experimentation. We introduce EXP-Bench, a novel benchmark designed to systematically evaluate AI agents on complete research experiments sourced from influential AI publications. Given a research question and incomplete starter code, EXP-Bench challenges AI agents to formulate hypotheses, design and implement experimental procedures, execute them, and analyze results. To enable the creation of such intricate and authentic tasks with high-fidelity, we design a semi-autonomous pipeline to extract and structure crucial experimental details from these research papers and their associated open source code. With the pipeline, EXP-Bench curated 461 AI research tasks from 51 top-tier AI research papers. Evaluations of leading LLM-based agents, such as OpenHands and IterativeAgents on EXP-Bench demonstrate partial capabilities: while scores on individual experimental aspects such as design or implementation correctness occasionally reach 20–35%, the success rate for complete, executable experiments was a mere 0.5%. By identifying these bottlenecks and providing realistic step-by-step experiment procedures, EXP-Bench serves as a vital tool for future AI agents to improve targeted research components and agent planning ability.
  }
}

@Article{cornserve:arxiv26,
  author          = {Jae-Won Chung and Jeff J. Ma and Jisang Ahn and Yizhuo Liang and Akshay Jajoo and Myungjin Lee and Mosharaf Chowdhury},
  title           = {Cornserve: A Distributed Serving System for Any-to-Any Multimodal Models},
  year            = {2026},
  month           = {Mar},
  volume          = {abs/2603.12118},
  archivePrefix   = {arXiv},
  eprint          = {2603.12118},
  url             = {https://arxiv.org/abs/2603.12118},
  publist_confkey = {arXiv:2603.12118},
  publist_link    = {paper || https://arxiv.org/abs/2603.12118},
  publist_topic   = {Systems + AI},
  publist_abstract = {
Any-to-Any models are an emerging class of multimodal models that accept combinations of multimodal data (e.g., text, image, video, audio) as input and generate them as output. Serving these models are challenging; different requests with different input and output modalities traverse different paths through the model computation graph, and each component of the model have different scaling characteristics.

We present Cornserve, a distributed serving system for generic Any-to-Any models. Cornserve provides a flexible task abstraction for expressing Any-to-Any model computation graphs, enabling component disaggregation and independent scaling. The distributed runtime dispatches compute to the data plane via an efficient record-and-replay execution model that keeps track of data dependencies, and forwards tensor data between components directly from the producer to the consumer. Built on Kubernetes with approximately 23K new lines of Python, Cornserve supports diverse Any-to-Any models and delivers up to 3.81x higher throughput and 5.79x lower tail latency.
  }
}