Add support for an overlayed Android root filesystem

This will allow users to extend the Android root filesystem with
additional system applications and changes.

scripts/container-manager.sh

@@ -10,6 +10,11 @@ DATA_PATH=$SNAP_COMMON/
 ROOTFS_PATH=$DATA_PATH/rootfs
 ANDROID_IMG=$SNAP/android.img
 
+if [ "$(id -u)" != 0 ]; then
+	echo "ERROR: You need to run the container manager as root"
+	exit 1
+fi
+
 if [ ! -e $ANDROID_IMG ]; then
 	echo "ERROR: android image does not exist"
 	exit 1
@@ -47,7 +52,14 @@ start() {
 		export ANBOX_LOG_LEVEL=debug
 	fi
 
+	EXTRA_ARGS=
+	enable_rootfs_overlay="$(snapctl get rootfs-overlay.enable)"
+	if [ "$enable_rootfs_overlay" = true ]; then
+		EXTRA_ARGS="$EXTRA_ARGS --use-rootfs-overlay"
+	fi
+
 	exec $AA_EXEC $SNAP/bin/anbox-wrapper.sh container-manager \
+		"$EXTRA_ARGS" \
 		--data-path=$DATA_PATH \
 		--android-image=$ANDROID_IMG \
 		--daemon

src/anbox/cli.h

@@ -71,7 +71,7 @@ std::ostream& operator<<(std::ostream& out,
 }
 
 // We are imposing size constraints to ensure a consistent CLI layout.
-typedef SizeConstrainedString<20> Name;
+typedef SizeConstrainedString<30> Name;
 typedef SizeConstrainedString<60> Usage;
 typedef SizeConstrainedString<100> Description;
 

src/anbox/cmds/container_manager.cpp

@@ -52,6 +52,9 @@ anbox::cmds::ContainerManager::ContainerManager()
   flag(cli::make_flag(cli::Name{"daemon"},
                       cli::Description{"Mark service as being started as systemd daemon"},
                       daemon_));
+  flag(cli::make_flag(cli::Name{"use-rootfs-overlay"},
+                      cli::Description{"Use an overlay for the Android rootfs"},
+                      enable_rootfs_overlay_));
 
   action([&](const cli::Command::Context&) {
     try {
@@ -87,7 +90,10 @@ anbox::cmds::ContainerManager::ContainerManager()
         return EXIT_FAILURE;
 
       auto rt = Runtime::create();
-      auto service = container::Service::create(rt, privileged_);
+      container::Service::Configuration config;
+      config.privileged = privileged_;
+      config.rootfs_overlay = enable_rootfs_overlay_;
+      auto service = container::Service::create(rt, config);
 
       rt->start();
       trap->run();
@@ -183,8 +189,17 @@ bool anbox::cmds::ContainerManager::setup_mounts() {
     return false;
   }
 
+  auto final_android_rootfs_dir = android_rootfs_dir;
+  if (enable_rootfs_overlay_) {
+    if (!setup_rootfs_overlay())
+      return false;
+
+    final_android_rootfs_dir = SystemConfiguration::instance().combined_rootfs_dir();
+  }
+
+
   for (const auto &dir_name : std::vector<std::string>{"cache", "data"}) {
-    auto target_dir_path = fs::path(android_rootfs_dir) / dir_name;
+    auto target_dir_path = fs::path(final_android_rootfs_dir) / dir_name;
     auto src_dir_path = SystemConfiguration::instance().data_dir() / dir_name;
 
     if (!fs::exists(src_dir_path)) {
@@ -214,3 +229,26 @@ bool anbox::cmds::ContainerManager::setup_mounts() {
 
   return true;
 }
+
+bool anbox::cmds::ContainerManager::setup_rootfs_overlay() {
+  const auto combined_rootfs_path = SystemConfiguration::instance().combined_rootfs_dir();
+  if (!fs::exists(combined_rootfs_path))
+    fs::create_directories(combined_rootfs_path);
+
+  const auto overlay_path = SystemConfiguration::instance().overlay_dir();
+  if (!fs::exists(overlay_path))
+    fs::create_directories(overlay_path);
+
+  const auto rootfs_path = SystemConfiguration::instance().rootfs_dir();
+  const auto overlay_config = utils::string_format("lowerdir=%s:%s", rootfs_path, overlay_path);
+  auto m = common::MountEntry::create("overlay", combined_rootfs_path, "overlay", MS_RDONLY, overlay_config.c_str());
+  if (!m) {
+    ERROR("Failed to setup rootfs overlay");
+    mounts_.clear();
+    return false;
+  }
+  mounts_.push_back(m);
+
+  DEBUG("Successfully setup rootfs overlay");
+  return true;
+}

src/anbox/cmds/container_manager.h

@@ -36,6 +36,7 @@ class ContainerManager : public cli::CommandWithFlagsAndAction {
 
  private:
   bool setup_mounts();
+  bool setup_rootfs_overlay();
 
   std::string android_img_path_;
   std::string data_path_;
@@ -43,6 +44,7 @@ class ContainerManager : public cli::CommandWithFlagsAndAction {
   std::vector<std::shared_ptr<common::MountEntry>> mounts_;
   bool privileged_ = false;
   bool daemon_ = false;
+  bool enable_rootfs_overlay_ = false;
 };
 }  // namespace cmds
 }  // namespace anbox

src/anbox/common/mount_entry.cpp

@@ -33,8 +33,12 @@ std::shared_ptr<MountEntry> MountEntry::create(const boost::filesystem::path &sr
   if (!data.empty())
     mount_data = reinterpret_cast<const void*>(data.c_str());
 
-  if (::mount(src.c_str(), target.c_str(), !fs_type.empty() ? fs_type.c_str() : nullptr, flags, mount_data) != 0)
+  DEBUG("Mounting %s on %s ...", src, target);
+
+  if (::mount(src.c_str(), target.c_str(), !fs_type.empty() ? fs_type.c_str() : nullptr, flags, mount_data) < 0) {
+    ERROR("Failed to mount %s: %s", target, strerror(errno));
     return nullptr;
+  }
 
   entry->active_ = true;
 

src/anbox/container/lxc_container.cpp

@@ -58,8 +58,12 @@ constexpr int device_minor(__dev_t dev) {
 
 namespace anbox {
 namespace container {
-LxcContainer::LxcContainer(bool privileged, const network::Credentials &creds)
-    : state_(State::inactive), container_(nullptr),  privileged_(privileged), creds_(creds) {
+LxcContainer::LxcContainer(bool privileged, bool rootfs_overlay, const network::Credentials &creds)
+    : state_(State::inactive),
+      container_(nullptr),
+      privileged_(privileged),
+      rootfs_overlay_(rootfs_overlay),
+      creds_(creds) {
   utils::ensure_paths({
       SystemConfiguration::instance().container_config_dir(),
       SystemConfiguration::instance().log_dir(),
@@ -68,7 +72,8 @@ LxcContainer::LxcContainer(bool privileged, const network::Credentials &creds)
 
 LxcContainer::~LxcContainer() {
   stop();
-  if (container_) lxc_container_put(container_);
+  if (container_)
+    lxc_container_put(container_);
 }
 
 void LxcContainer::setup_id_map() {
@@ -261,7 +266,10 @@ void LxcContainer::start(const Configuration &configuration) {
 
   set_config_item("lxc.init.cmd", "/anbox-init.sh");
 
-  const auto rootfs_path = SystemConfiguration::instance().rootfs_dir();
+  auto rootfs_path = SystemConfiguration::instance().rootfs_dir();
+  if (rootfs_overlay_)
+    rootfs_path = SystemConfiguration::instance().combined_rootfs_dir();
+
   DEBUG("Using rootfs path %s", rootfs_path);
   set_config_item("lxc.rootfs.path", rootfs_path);
 

src/anbox/container/lxc_container.h

@@ -29,7 +29,7 @@ namespace anbox {
 namespace container {
 class LxcContainer : public Container {
  public:
-  LxcContainer(bool privileged, const network::Credentials &creds);
+  LxcContainer(bool privileged, bool rootfs_overlay, const network::Credentials &creds);
   ~LxcContainer();
 
   void start(const Configuration &configuration) override;
@@ -45,6 +45,7 @@ class LxcContainer : public Container {
   State state_;
   lxc_container *container_;
   bool privileged_;
+  bool rootfs_overlay_;
   network::Credentials creds_;
 };
 }  // namespace container

src/anbox/container/service.cpp

@@ -34,8 +34,8 @@ namespace fs = boost::filesystem;
 
 namespace anbox {
 namespace container {
-std::shared_ptr<Service> Service::create(const std::shared_ptr<Runtime> &rt, bool privileged) {
-  auto sp = std::shared_ptr<Service>(new Service(rt, privileged));
+std::shared_ptr<Service> Service::create(const std::shared_ptr<Runtime> &rt, const Configuration &config) {
+  auto sp = std::shared_ptr<Service>(new Service(rt, config));
 
   auto wp = std::weak_ptr<Service>(sp);
   auto delegate_connector = std::make_shared<network::DelegateConnectionCreator<boost::asio::local::stream_protocol>>(
@@ -59,11 +59,11 @@ std::shared_ptr<Service> Service::create(const std::shared_ptr<Runtime> &rt, boo
   return sp;
 }
 
-Service::Service(const std::shared_ptr<Runtime> &rt, bool privileged)
+Service::Service(const std::shared_ptr<Runtime> &rt, const Configuration &config)
     : dispatcher_(anbox::common::create_dispatcher_for_runtime(rt)),
       next_connection_id_(0),
       connections_(std::make_shared<network::Connections<network::SocketConnection>>()),
-      privileged_(privileged) {
+      config_(config) {
 }
 
 Service::~Service() {
@@ -86,7 +86,7 @@ void Service::new_client(std::shared_ptr<boost::asio::local::stream_protocol::so
   auto pending_calls = std::make_shared<rpc::PendingCallCache>();
   auto rpc_channel = std::make_shared<rpc::Channel>(pending_calls, messenger);
   auto server = std::make_shared<container::ManagementApiSkeleton>(
-      pending_calls, std::make_shared<LxcContainer>(privileged_, messenger->creds()));
+      pending_calls, std::make_shared<LxcContainer>(config_.privileged, config_.rootfs_overlay, messenger->creds()));
   auto processor = std::make_shared<container::ManagementApiMessageProcessor>(
       messenger, pending_calls, server);
 

src/anbox/container/service.h

@@ -30,12 +30,18 @@ namespace anbox {
 namespace container {
 class Service : public std::enable_shared_from_this<Service> {
  public:
-  static std::shared_ptr<Service> create(const std::shared_ptr<Runtime> &rt, bool privileged);
+  struct Configuration {
+    bool privileged = false;
+    bool rootfs_overlay = true;
+  };
+
+  static std::shared_ptr<Service> create(const std::shared_ptr<Runtime> &rt,
+                                         const Configuration &config);
 
   ~Service();
 
  private:
-  Service(const std::shared_ptr<Runtime> &rt, bool privileged);
+  Service(const std::shared_ptr<Runtime> &rt, const Configuration &config);
 
   int next_id();
   void new_client(std::shared_ptr<
@@ -46,7 +52,7 @@ class Service : public std::enable_shared_from_this<Service> {
   std::atomic<int> next_connection_id_;
   std::shared_ptr<network::Connections<network::SocketConnection>> connections_;
   std::shared_ptr<Container> backend_;
-  bool privileged_;
+  Configuration config_;
 };
 }  // namespace container
 }  // namespace anbox

src/anbox/system_configuration.cpp

@@ -55,6 +55,14 @@ std::string anbox::SystemConfiguration::rootfs_dir() const {
   return (data_path / "rootfs").string();
 }
 
+std::string anbox::SystemConfiguration::overlay_dir() const {
+  return (data_path / "rootfs-overlay").string();
+}
+
+std::string anbox::SystemConfiguration::combined_rootfs_dir() const {
+  return (data_path / "combined-rootfs").string();
+}
+
 std::string anbox::SystemConfiguration::log_dir() const {
   return (data_path / "logs").string();
 }

src/anbox/system_configuration.h

@@ -34,6 +34,8 @@ class SystemConfiguration {
 
   boost::filesystem::path data_dir() const;
   std::string rootfs_dir() const;
+  std::string overlay_dir() const;
+  std::string combined_rootfs_dir() const;
   std::string log_dir() const;
   std::string socket_dir() const;
   std::string container_config_dir() const;